835 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:"2; RUN: opt -passes=loop-vectorize -mcpu=knl -S %s | FileCheck %s -check-prefix=AVX5123; RUN: opt -passes=loop-vectorize -mcpu=knl -force-vector-width=2 -force-target-max-vector-interleave=1 -S %s | FileCheck %s -check-prefix=FVW24 5; With a force-vector-width, it is sometimes more profitable to generate6; scalarized and predicated stores instead of masked scatter. Disable7; interleaving to simplify CHECKs in that scenario.8 9target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"10target triple = "x86_64-pc_linux"11 12; The source code:13;14;void foo1(ptr __restrict__ in, ptr __restrict__ out, int * __restrict__ trigger, int * __restrict__ index) {15;16; for (int i=0; i < SIZE; ++i) {17; if (trigger[i] > 0) {18; out[i] = in[index[i]] + (float) 0.5;19; }20; }21;}22 23; Function Attrs: nounwind uwtable24define void @foo1(ptr noalias %in, ptr noalias %out, ptr noalias %trigger, ptr noalias %index) {25; AVX512-LABEL: @foo1(26; AVX512-NEXT: entry:27; AVX512-NEXT: br label [[ENTRY:%.*]]28; AVX512: vector.ph:29; AVX512-NEXT: br label [[VECTOR_BODY:%.*]]30; AVX512: vector.body:31; AVX512-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]32; AVX512-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER:%.*]], i64 [[INDEX1]]33; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i32>, ptr [[TMP1]], align 434; AVX512-NEXT: [[TMP3:%.*]] = icmp sgt <16 x i32> [[WIDE_LOAD]], zeroinitializer35; AVX512-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[INDEX:%.*]], i64 [[INDEX1]]36; AVX512-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr align 4 [[TMP4]], <16 x i1> [[TMP3]], <16 x i32> poison)37; AVX512-NEXT: [[TMP6:%.*]] = sext <16 x i32> [[WIDE_MASKED_LOAD]] to <16 x i64>38; AVX512-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[IN:%.*]], <16 x i64> [[TMP6]]39; AVX512-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> align 4 [[TMP7]], <16 x i1> [[TMP3]], <16 x float> poison)40; AVX512-NEXT: [[TMP8:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER]], splat (float 5.000000e-01)41; AVX512-NEXT: [[TMP9:%.*]] = getelementptr float, ptr [[OUT:%.*]], i64 [[INDEX1]]42; AVX512-NEXT: call void @llvm.masked.store.v16f32.p0(<16 x float> [[TMP8]], ptr align 4 [[TMP9]], <16 x i1> [[TMP3]])43; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 1644; AVX512-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 409645; AVX512-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]46; AVX512: middle.block:47; AVX512-NEXT: br label [[FOR_END:%.*]]48; AVX512: for.end:49; AVX512-NEXT: ret void50;51; FVW2-LABEL: @foo1(52; FVW2-NEXT: entry:53; FVW2-NEXT: br label [[ENTRY:%.*]]54; FVW2: vector.ph:55; FVW2-NEXT: br label [[VECTOR_BODY:%.*]]56; FVW2: vector.body:57; FVW2-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]58; FVW2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER:%.*]], i64 [[INDEX1]]59; FVW2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 460; FVW2-NEXT: [[TMP3:%.*]] = icmp sgt <2 x i32> [[WIDE_LOAD]], zeroinitializer61; FVW2-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[INDEX:%.*]], i64 [[INDEX1]]62; FVW2-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr align 4 [[TMP4]], <2 x i1> [[TMP3]], <2 x i32> poison)63; FVW2-NEXT: [[TMP6:%.*]] = sext <2 x i32> [[WIDE_MASKED_LOAD]] to <2 x i64>64; FVW2-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[IN:%.*]], <2 x i64> [[TMP6]]65; FVW2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> align 4 [[TMP7]], <2 x i1> [[TMP3]], <2 x float> poison)66; FVW2-NEXT: [[TMP8:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER]], splat (float 5.000000e-01)67; FVW2-NEXT: [[TMP9:%.*]] = getelementptr float, ptr [[OUT:%.*]], i64 [[INDEX1]]68; FVW2-NEXT: call void @llvm.masked.store.v2f32.p0(<2 x float> [[TMP8]], ptr align 4 [[TMP9]], <2 x i1> [[TMP3]])69; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 270; FVW2-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 409671; FVW2-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]72; FVW2: middle.block:73; FVW2-NEXT: br label [[FOR_END:%.*]]74; FVW2: for.end:75; FVW2-NEXT: ret void76;77entry:78 br label %for.body79 80for.body:81 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]82 %arrayidx = getelementptr inbounds i32, ptr %trigger, i64 %indvars.iv83 %0 = load i32, ptr %arrayidx, align 484 %cmp1 = icmp sgt i32 %0, 085 br i1 %cmp1, label %if.then, label %for.inc86 87if.then:88 %arrayidx3 = getelementptr inbounds i32, ptr %index, i64 %indvars.iv89 %1 = load i32, ptr %arrayidx3, align 490 %idxprom4 = sext i32 %1 to i6491 %arrayidx5 = getelementptr inbounds float, ptr %in, i64 %idxprom492 %2 = load float, ptr %arrayidx5, align 493 %add = fadd float %2, 5.000000e-0194 %arrayidx7 = getelementptr inbounds float, ptr %out, i64 %indvars.iv95 store float %add, ptr %arrayidx7, align 496 br label %for.inc97 98for.inc:99 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1100 %exitcond.not = icmp eq i64 %indvars.iv.next, 4096101 br i1 %exitcond.not, label %for.end, label %for.body102 103for.end:104 ret void105}106 107; The source code108;void foo2 (In * __restrict__ in, ptr __restrict__ out, int * __restrict__ trigger) {109;110; for (int i=0; i<SIZE; i += 16) {111; if (trigger[i] > 0) {112; out[i] = in[i].b + (float) 0.5;113; }114; }115;}116 117%struct.In = type { float, float }118 119define void @foo2(ptr noalias %in, ptr noalias %out, ptr noalias %trigger, ptr noalias %index) #0 {120; AVX512-LABEL: @foo2(121; AVX512-NEXT: entry:122; AVX512-NEXT: br label [[ENTRY:%.*]]123; AVX512: vector.ph:124; AVX512-NEXT: br label [[VECTOR_BODY:%.*]]125; AVX512: vector.body:126; AVX512-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]127; AVX512-NEXT: [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]128; AVX512-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER:%.*]], <16 x i64> [[VEC_IND]]129; AVX512-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> align 4 [[TMP0]], <16 x i1> splat (i1 true), <16 x i32> poison)130; AVX512-NEXT: [[TMP1:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER]], zeroinitializer131; AVX512-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], ptr [[IN:%.*]], <16 x i64> [[VEC_IND]], i32 1132; AVX512-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> align 4 [[TMP2]], <16 x i1> [[TMP1]], <16 x float> poison)133; AVX512-NEXT: [[TMP3:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER2]], splat (float 5.000000e-01)134; AVX512-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[OUT:%.*]], <16 x i64> [[VEC_IND]]135; AVX512-NEXT: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> [[TMP3]], <16 x ptr> align 4 [[TMP4]], <16 x i1> [[TMP1]])136; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 16137; AVX512-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <16 x i64> [[VEC_IND]], splat (i64 256)138; AVX512-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256139; AVX512-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]140; AVX512: middle.block:141; AVX512-NEXT: br label [[FOR_END:%.*]]142; AVX512: for.end:143; AVX512-NEXT: ret void144;145; FVW2-LABEL: @foo2(146; FVW2-NEXT: entry:147; FVW2-NEXT: br label [[ENTRY:%.*]]148; FVW2: vector.ph:149; FVW2-NEXT: br label [[VECTOR_BODY:%.*]]150; FVW2: vector.body:151; FVW2-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]152; FVW2-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 16>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE3]] ]153; FVW2-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX1]], 16154; FVW2-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0155; FVW2-NEXT: [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 16156; FVW2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER:%.*]], i64 [[TMP0]]157; FVW2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[TMP1]]158; FVW2-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 4159; FVW2-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP3]], align 4160; FVW2-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i32 0161; FVW2-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP5]], i32 1162; FVW2-NEXT: [[TMP8:%.*]] = icmp sgt <2 x i32> [[TMP7]], zeroinitializer163; FVW2-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], ptr [[IN:%.*]], <2 x i64> [[VEC_IND]], i32 1164; FVW2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> align 4 [[TMP9]], <2 x i1> [[TMP8]], <2 x float> poison)165; FVW2-NEXT: [[TMP10:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER]], splat (float 5.000000e-01)166; FVW2-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP8]], i32 0167; FVW2-NEXT: br i1 [[TMP11]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]168; FVW2: pred.store.if:169; FVW2-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[OUT:%.*]], i64 [[TMP0]]170; FVW2-NEXT: [[TMP13:%.*]] = extractelement <2 x float> [[TMP10]], i32 0171; FVW2-NEXT: store float [[TMP13]], ptr [[TMP12]], align 4172; FVW2-NEXT: br label [[PRED_STORE_CONTINUE]]173; FVW2: pred.store.continue:174; FVW2-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[TMP8]], i32 1175; FVW2-NEXT: br i1 [[TMP14]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]176; FVW2: pred.store.if2:177; FVW2-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[OUT]], i64 [[TMP1]]178; FVW2-NEXT: [[TMP16:%.*]] = extractelement <2 x float> [[TMP10]], i32 1179; FVW2-NEXT: store float [[TMP16]], ptr [[TMP15]], align 4180; FVW2-NEXT: br label [[PRED_STORE_CONTINUE3]]181; FVW2: pred.store.continue3:182; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 2183; FVW2-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 32)184; FVW2-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256185; FVW2-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]186; FVW2: middle.block:187; FVW2-NEXT: br label [[FOR_END:%.*]]188; FVW2: for.end:189; FVW2-NEXT: ret void190;191entry:192 br label %for.body193 194for.body:195 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]196 %arrayidx = getelementptr inbounds i32, ptr %trigger, i64 %indvars.iv197 %0 = load i32, ptr %arrayidx, align 4198 %cmp1 = icmp sgt i32 %0, 0199 br i1 %cmp1, label %if.then, label %for.inc200 201if.then:202 %b = getelementptr inbounds %struct.In, ptr %in, i64 %indvars.iv, i32 1203 %1 = load float, ptr %b, align 4204 %add = fadd float %1, 5.000000e-01205 %arrayidx5 = getelementptr inbounds float, ptr %out, i64 %indvars.iv206 store float %add, ptr %arrayidx5, align 4207 br label %for.inc208 209for.inc:210 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 16211 %cmp = icmp ult i64 %indvars.iv, 4080212 br i1 %cmp, label %for.body, label %for.end213 214for.end:215 ret void216}217 218; The source code219;struct Out {220; float a;221; float b;222;};223;void foo3 (In * __restrict__ in, Out * __restrict__ out, int * __restrict__ trigger) {224;225; for (int i=0; i<SIZE; i += 16) {226; if (trigger[i] > 0) {227; out[i].b = in[i].b + (float) 0.5;228; }229; }230;}231 232%struct.Out = type { float, float }233 234define void @foo3(ptr noalias %in, ptr noalias %out, ptr noalias %trigger) {235; AVX512-LABEL: @foo3(236; AVX512-NEXT: entry:237; AVX512-NEXT: br label [[ENTRY:%.*]]238; AVX512: vector.ph:239; AVX512-NEXT: br label [[VECTOR_BODY:%.*]]240; AVX512: vector.body:241; AVX512-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]242; AVX512-NEXT: [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]243; AVX512-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER:%.*]], <16 x i64> [[VEC_IND]]244; AVX512-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> align 4 [[TMP0]], <16 x i1> splat (i1 true), <16 x i32> poison)245; AVX512-NEXT: [[TMP1:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER]], zeroinitializer246; AVX512-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], ptr [[IN:%.*]], <16 x i64> [[VEC_IND]], i32 1247; AVX512-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> align 4 [[TMP2]], <16 x i1> [[TMP1]], <16 x float> poison)248; AVX512-NEXT: [[TMP3:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER1]], splat (float 5.000000e-01)249; AVX512-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_OUT:%.*]], ptr [[OUT:%.*]], <16 x i64> [[VEC_IND]], i32 1250; AVX512-NEXT: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> [[TMP3]], <16 x ptr> align 4 [[TMP4]], <16 x i1> [[TMP1]])251; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16252; AVX512-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <16 x i64> [[VEC_IND]], splat (i64 256)253; AVX512-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256254; AVX512-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]255; AVX512: middle.block:256; AVX512-NEXT: br label [[FOR_END:%.*]]257; AVX512: for.end:258; AVX512-NEXT: ret void259;260; FVW2-LABEL: @foo3(261; FVW2-NEXT: entry:262; FVW2-NEXT: br label [[ENTRY:%.*]]263; FVW2: vector.ph:264; FVW2-NEXT: br label [[VECTOR_BODY:%.*]]265; FVW2: vector.body:266; FVW2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]267; FVW2-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 16>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE2]] ]268; FVW2-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 16269; FVW2-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0270; FVW2-NEXT: [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 16271; FVW2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER:%.*]], i64 [[TMP0]]272; FVW2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[TMP1]]273; FVW2-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 4274; FVW2-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP3]], align 4275; FVW2-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i32 0276; FVW2-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP5]], i32 1277; FVW2-NEXT: [[TMP8:%.*]] = icmp sgt <2 x i32> [[TMP7]], zeroinitializer278; FVW2-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], ptr [[IN:%.*]], <2 x i64> [[VEC_IND]], i32 1279; FVW2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> align 4 [[TMP9]], <2 x i1> [[TMP8]], <2 x float> poison)280; FVW2-NEXT: [[TMP10:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER]], splat (float 5.000000e-01)281; FVW2-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP8]], i32 0282; FVW2-NEXT: br i1 [[TMP11]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]283; FVW2: pred.store.if:284; FVW2-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_OUT:%.*]], ptr [[OUT:%.*]], i64 [[TMP0]], i32 1285; FVW2-NEXT: [[TMP13:%.*]] = extractelement <2 x float> [[TMP10]], i32 0286; FVW2-NEXT: store float [[TMP13]], ptr [[TMP12]], align 4287; FVW2-NEXT: br label [[PRED_STORE_CONTINUE]]288; FVW2: pred.store.continue:289; FVW2-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[TMP8]], i32 1290; FVW2-NEXT: br i1 [[TMP14]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]291; FVW2: pred.store.if1:292; FVW2-NEXT: [[TMP15:%.*]] = getelementptr inbounds [[STRUCT_OUT]], ptr [[OUT]], i64 [[TMP1]], i32 1293; FVW2-NEXT: [[TMP16:%.*]] = extractelement <2 x float> [[TMP10]], i32 1294; FVW2-NEXT: store float [[TMP16]], ptr [[TMP15]], align 4295; FVW2-NEXT: br label [[PRED_STORE_CONTINUE2]]296; FVW2: pred.store.continue2:297; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2298; FVW2-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 32)299; FVW2-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256300; FVW2-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]301; FVW2: middle.block:302; FVW2-NEXT: br label [[FOR_END:%.*]]303; FVW2: for.end:304; FVW2-NEXT: ret void305;306entry:307 br label %for.body308 309for.body:310 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]311 %arrayidx = getelementptr inbounds i32, ptr %trigger, i64 %indvars.iv312 %0 = load i32, ptr %arrayidx, align 4313 %cmp1 = icmp sgt i32 %0, 0314 br i1 %cmp1, label %if.then, label %for.inc315 316if.then:317 %b = getelementptr inbounds %struct.In, ptr %in, i64 %indvars.iv, i32 1318 %1 = load float, ptr %b, align 4319 %add = fadd float %1, 5.000000e-01320 %b6 = getelementptr inbounds %struct.Out, ptr %out, i64 %indvars.iv, i32 1321 store float %add, ptr %b6, align 4322 br label %for.inc323 324for.inc:325 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 16326 %cmp = icmp ult i64 %indvars.iv, 4080327 br i1 %cmp, label %for.body, label %for.end328 329for.end:330 ret void331}332declare void @llvm.masked.scatter.v16f32.v16p0(<16 x float>, <16 x ptr>, i32, <16 x i1>)333 334; The same as @foo2 but scatter/gather argument is a vecotr of ptrs with addresspace 1335 336define void @foo2_addrspace(ptr addrspace(1) noalias %in, ptr addrspace(1) noalias %out, ptr noalias %trigger, ptr noalias %index) #0 {337; AVX512-LABEL: @foo2_addrspace(338; AVX512-NEXT: entry:339; AVX512-NEXT: br label [[ENTRY:%.*]]340; AVX512: vector.ph:341; AVX512-NEXT: br label [[VECTOR_BODY:%.*]]342; AVX512: vector.body:343; AVX512-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]344; AVX512-NEXT: [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]345; AVX512-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER:%.*]], <16 x i64> [[VEC_IND]]346; AVX512-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> align 4 [[TMP0]], <16 x i1> splat (i1 true), <16 x i32> poison)347; AVX512-NEXT: [[TMP1:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER]], zeroinitializer348; AVX512-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], ptr addrspace(1) [[IN:%.*]], <16 x i64> [[VEC_IND]], i32 1349; AVX512-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1(<16 x ptr addrspace(1)> align 4 [[TMP2]], <16 x i1> [[TMP1]], <16 x float> poison)350; AVX512-NEXT: [[TMP3:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER2]], splat (float 5.000000e-01)351; AVX512-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT:%.*]], <16 x i64> [[VEC_IND]]352; AVX512-NEXT: call void @llvm.masked.scatter.v16f32.v16p1(<16 x float> [[TMP3]], <16 x ptr addrspace(1)> align 4 [[TMP4]], <16 x i1> [[TMP1]])353; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 16354; AVX512-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <16 x i64> [[VEC_IND]], splat (i64 256)355; AVX512-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256356; AVX512-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]357; AVX512: middle.block:358; AVX512-NEXT: br label [[FOR_END:%.*]]359; AVX512: for.end:360; AVX512-NEXT: ret void361;362; FVW2-LABEL: @foo2_addrspace(363; FVW2-NEXT: entry:364; FVW2-NEXT: br label [[ENTRY:%.*]]365; FVW2: vector.ph:366; FVW2-NEXT: br label [[VECTOR_BODY:%.*]]367; FVW2: vector.body:368; FVW2-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]369; FVW2-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 16>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE3]] ]370; FVW2-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX1]], 16371; FVW2-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0372; FVW2-NEXT: [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 16373; FVW2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER:%.*]], i64 [[TMP0]]374; FVW2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[TMP1]]375; FVW2-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 4376; FVW2-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP3]], align 4377; FVW2-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i32 0378; FVW2-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP5]], i32 1379; FVW2-NEXT: [[TMP8:%.*]] = icmp sgt <2 x i32> [[TMP7]], zeroinitializer380; FVW2-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], ptr addrspace(1) [[IN:%.*]], <2 x i64> [[VEC_IND]], i32 1381; FVW2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p1(<2 x ptr addrspace(1)> align 4 [[TMP9]], <2 x i1> [[TMP8]], <2 x float> poison)382; FVW2-NEXT: [[TMP10:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER]], splat (float 5.000000e-01)383; FVW2-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP8]], i32 0384; FVW2-NEXT: br i1 [[TMP11]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]385; FVW2: pred.store.if:386; FVW2-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT:%.*]], i64 [[TMP0]]387; FVW2-NEXT: [[TMP13:%.*]] = extractelement <2 x float> [[TMP10]], i32 0388; FVW2-NEXT: store float [[TMP13]], ptr addrspace(1) [[TMP12]], align 4389; FVW2-NEXT: br label [[PRED_STORE_CONTINUE]]390; FVW2: pred.store.continue:391; FVW2-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[TMP8]], i32 1392; FVW2-NEXT: br i1 [[TMP14]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]393; FVW2: pred.store.if2:394; FVW2-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 [[TMP1]]395; FVW2-NEXT: [[TMP16:%.*]] = extractelement <2 x float> [[TMP10]], i32 1396; FVW2-NEXT: store float [[TMP16]], ptr addrspace(1) [[TMP15]], align 4397; FVW2-NEXT: br label [[PRED_STORE_CONTINUE3]]398; FVW2: pred.store.continue3:399; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 2400; FVW2-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 32)401; FVW2-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256402; FVW2-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]403; FVW2: middle.block:404; FVW2-NEXT: br label [[FOR_END:%.*]]405; FVW2: for.end:406; FVW2-NEXT: ret void407;408entry:409 br label %for.body410 411for.body:412 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]413 %arrayidx = getelementptr inbounds i32, ptr %trigger, i64 %indvars.iv414 %0 = load i32, ptr %arrayidx, align 4415 %cmp1 = icmp sgt i32 %0, 0416 br i1 %cmp1, label %if.then, label %for.inc417 418if.then:419 %b = getelementptr inbounds %struct.In, ptr addrspace(1) %in, i64 %indvars.iv, i32 1420 %1 = load float, ptr addrspace(1) %b, align 4421 %add = fadd float %1, 5.000000e-01422 %arrayidx5 = getelementptr inbounds float, ptr addrspace(1) %out, i64 %indvars.iv423 store float %add, ptr addrspace(1) %arrayidx5, align 4424 br label %for.inc425 426for.inc:427 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 16428 %cmp = icmp ult i64 %indvars.iv, 4080429 br i1 %cmp, label %for.body, label %for.end430 431for.end:432 ret void433}434 435; Same as foo2_addrspace but here only the input has the non-default address space.436 437define void @foo2_addrspace2(ptr addrspace(1) noalias %in, ptr addrspace(0) noalias %out, ptr noalias %trigger, ptr noalias %index) {438; AVX512-LABEL: @foo2_addrspace2(439; AVX512-NEXT: entry:440; AVX512-NEXT: br label [[ENTRY:%.*]]441; AVX512: vector.ph:442; AVX512-NEXT: br label [[VECTOR_BODY:%.*]]443; AVX512: vector.body:444; AVX512-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]445; AVX512-NEXT: [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]446; AVX512-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER:%.*]], <16 x i64> [[VEC_IND]]447; AVX512-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> align 4 [[TMP0]], <16 x i1> splat (i1 true), <16 x i32> poison)448; AVX512-NEXT: [[TMP1:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER]], zeroinitializer449; AVX512-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], ptr addrspace(1) [[IN:%.*]], <16 x i64> [[VEC_IND]], i32 1450; AVX512-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1(<16 x ptr addrspace(1)> align 4 [[TMP2]], <16 x i1> [[TMP1]], <16 x float> poison)451; AVX512-NEXT: [[TMP3:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER2]], splat (float 5.000000e-01)452; AVX512-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[OUT:%.*]], <16 x i64> [[VEC_IND]]453; AVX512-NEXT: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> [[TMP3]], <16 x ptr> align 4 [[TMP4]], <16 x i1> [[TMP1]])454; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 16455; AVX512-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <16 x i64> [[VEC_IND]], splat (i64 256)456; AVX512-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256457; AVX512-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]458; AVX512: middle.block:459; AVX512-NEXT: br label [[FOR_END:%.*]]460; AVX512: for.end:461; AVX512-NEXT: ret void462;463; FVW2-LABEL: @foo2_addrspace2(464; FVW2-NEXT: entry:465; FVW2-NEXT: br label [[ENTRY:%.*]]466; FVW2: vector.ph:467; FVW2-NEXT: br label [[VECTOR_BODY:%.*]]468; FVW2: vector.body:469; FVW2-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]470; FVW2-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 16>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE3]] ]471; FVW2-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX1]], 16472; FVW2-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0473; FVW2-NEXT: [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 16474; FVW2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER:%.*]], i64 [[TMP0]]475; FVW2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[TMP1]]476; FVW2-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 4477; FVW2-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP3]], align 4478; FVW2-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i32 0479; FVW2-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP5]], i32 1480; FVW2-NEXT: [[TMP8:%.*]] = icmp sgt <2 x i32> [[TMP7]], zeroinitializer481; FVW2-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], ptr addrspace(1) [[IN:%.*]], <2 x i64> [[VEC_IND]], i32 1482; FVW2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p1(<2 x ptr addrspace(1)> align 4 [[TMP9]], <2 x i1> [[TMP8]], <2 x float> poison)483; FVW2-NEXT: [[TMP10:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER]], splat (float 5.000000e-01)484; FVW2-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP8]], i32 0485; FVW2-NEXT: br i1 [[TMP11]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]486; FVW2: pred.store.if:487; FVW2-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[OUT:%.*]], i64 [[TMP0]]488; FVW2-NEXT: [[TMP13:%.*]] = extractelement <2 x float> [[TMP10]], i32 0489; FVW2-NEXT: store float [[TMP13]], ptr [[TMP12]], align 4490; FVW2-NEXT: br label [[PRED_STORE_CONTINUE]]491; FVW2: pred.store.continue:492; FVW2-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[TMP8]], i32 1493; FVW2-NEXT: br i1 [[TMP14]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]494; FVW2: pred.store.if2:495; FVW2-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[OUT]], i64 [[TMP1]]496; FVW2-NEXT: [[TMP16:%.*]] = extractelement <2 x float> [[TMP10]], i32 1497; FVW2-NEXT: store float [[TMP16]], ptr [[TMP15]], align 4498; FVW2-NEXT: br label [[PRED_STORE_CONTINUE3]]499; FVW2: pred.store.continue3:500; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 2501; FVW2-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 32)502; FVW2-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256503; FVW2-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]504; FVW2: middle.block:505; FVW2-NEXT: br label [[FOR_END:%.*]]506; FVW2: for.end:507; FVW2-NEXT: ret void508;509entry:510 br label %for.body511 512for.body:513 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]514 %arrayidx = getelementptr inbounds i32, ptr %trigger, i64 %indvars.iv515 %0 = load i32, ptr %arrayidx, align 4516 %cmp1 = icmp sgt i32 %0, 0517 br i1 %cmp1, label %if.then, label %for.inc518 519if.then:520 %b = getelementptr inbounds %struct.In, ptr addrspace(1) %in, i64 %indvars.iv, i32 1521 %1 = load float, ptr addrspace(1) %b, align 4522 %add = fadd float %1, 5.000000e-01523 %arrayidx5 = getelementptr inbounds float, ptr %out, i64 %indvars.iv524 store float %add, ptr %arrayidx5, align 4525 br label %for.inc526 527for.inc:528 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 16529 %cmp = icmp ult i64 %indvars.iv, 4080530 br i1 %cmp, label %for.body, label %for.end531 532for.end:533 ret void534}535 536; Same as foo2_addrspace but here only the output has the non-default address space.537 538define void @foo2_addrspace3(ptr addrspace(0) noalias %in, ptr addrspace(1) noalias %out, ptr noalias %trigger, ptr noalias %index) {539; AVX512-LABEL: @foo2_addrspace3(540; AVX512-NEXT: entry:541; AVX512-NEXT: br label [[ENTRY:%.*]]542; AVX512: vector.ph:543; AVX512-NEXT: br label [[VECTOR_BODY:%.*]]544; AVX512: vector.body:545; AVX512-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]546; AVX512-NEXT: [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]547; AVX512-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER:%.*]], <16 x i64> [[VEC_IND]]548; AVX512-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> align 4 [[TMP0]], <16 x i1> splat (i1 true), <16 x i32> poison)549; AVX512-NEXT: [[TMP1:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER]], zeroinitializer550; AVX512-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], ptr [[IN:%.*]], <16 x i64> [[VEC_IND]], i32 1551; AVX512-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> align 4 [[TMP2]], <16 x i1> [[TMP1]], <16 x float> poison)552; AVX512-NEXT: [[TMP3:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER2]], splat (float 5.000000e-01)553; AVX512-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT:%.*]], <16 x i64> [[VEC_IND]]554; AVX512-NEXT: call void @llvm.masked.scatter.v16f32.v16p1(<16 x float> [[TMP3]], <16 x ptr addrspace(1)> align 4 [[TMP4]], <16 x i1> [[TMP1]])555; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 16556; AVX512-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <16 x i64> [[VEC_IND]], splat (i64 256)557; AVX512-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256558; AVX512-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]559; AVX512: middle.block:560; AVX512-NEXT: br label [[FOR_END:%.*]]561; AVX512: for.end:562; AVX512-NEXT: ret void563;564; FVW2-LABEL: @foo2_addrspace3(565; FVW2-NEXT: entry:566; FVW2-NEXT: br label [[ENTRY:%.*]]567; FVW2: vector.ph:568; FVW2-NEXT: br label [[VECTOR_BODY:%.*]]569; FVW2: vector.body:570; FVW2-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]571; FVW2-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 16>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE3]] ]572; FVW2-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX1]], 16573; FVW2-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0574; FVW2-NEXT: [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 16575; FVW2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER:%.*]], i64 [[TMP0]]576; FVW2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[TMP1]]577; FVW2-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 4578; FVW2-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP3]], align 4579; FVW2-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i32 0580; FVW2-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP5]], i32 1581; FVW2-NEXT: [[TMP8:%.*]] = icmp sgt <2 x i32> [[TMP7]], zeroinitializer582; FVW2-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], ptr [[IN:%.*]], <2 x i64> [[VEC_IND]], i32 1583; FVW2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> align 4 [[TMP9]], <2 x i1> [[TMP8]], <2 x float> poison)584; FVW2-NEXT: [[TMP10:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER]], splat (float 5.000000e-01)585; FVW2-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP8]], i32 0586; FVW2-NEXT: br i1 [[TMP11]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]587; FVW2: pred.store.if:588; FVW2-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT:%.*]], i64 [[TMP0]]589; FVW2-NEXT: [[TMP13:%.*]] = extractelement <2 x float> [[TMP10]], i32 0590; FVW2-NEXT: store float [[TMP13]], ptr addrspace(1) [[TMP12]], align 4591; FVW2-NEXT: br label [[PRED_STORE_CONTINUE]]592; FVW2: pred.store.continue:593; FVW2-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[TMP8]], i32 1594; FVW2-NEXT: br i1 [[TMP14]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]595; FVW2: pred.store.if2:596; FVW2-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 [[TMP1]]597; FVW2-NEXT: [[TMP16:%.*]] = extractelement <2 x float> [[TMP10]], i32 1598; FVW2-NEXT: store float [[TMP16]], ptr addrspace(1) [[TMP15]], align 4599; FVW2-NEXT: br label [[PRED_STORE_CONTINUE3]]600; FVW2: pred.store.continue3:601; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 2602; FVW2-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 32)603; FVW2-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256604; FVW2-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]605; FVW2: middle.block:606; FVW2-NEXT: br label [[FOR_END:%.*]]607; FVW2: for.end:608; FVW2-NEXT: ret void609;610entry:611 br label %for.body612 613for.body:614 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]615 %arrayidx = getelementptr inbounds i32, ptr %trigger, i64 %indvars.iv616 %0 = load i32, ptr %arrayidx, align 4617 %cmp1 = icmp sgt i32 %0, 0618 br i1 %cmp1, label %if.then, label %for.inc619 620if.then:621 %b = getelementptr inbounds %struct.In, ptr %in, i64 %indvars.iv, i32 1622 %1 = load float, ptr %b, align 4623 %add = fadd float %1, 5.000000e-01624 %arrayidx5 = getelementptr inbounds float, ptr addrspace(1) %out, i64 %indvars.iv625 store float %add, ptr addrspace(1) %arrayidx5, align 4626 br label %for.inc627 628for.inc:629 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 16630 %cmp = icmp ult i64 %indvars.iv, 4080631 br i1 %cmp, label %for.body, label %for.end632 633for.end:634 ret void635}636 637; Using gathers is not profitable for this function. PR48429.638define void @test_gather_not_profitable_pr48429(i32 %d, ptr readonly noalias %ptr, ptr nocapture noalias %dest) {639; AVX512-LABEL: @test_gather_not_profitable_pr48429(640; AVX512-NEXT: entry:641; AVX512-NEXT: [[IDX_EXT:%.*]] = sext i32 [[D:%.*]] to i64642; AVX512-NEXT: [[SCEVGEP1:%.*]] = getelementptr float, ptr [[PTR:%.*]], i64 [[IDX_EXT]]643; AVX512-NEXT: [[CMP_NOT10:%.*]] = icmp eq i32 [[D]], 0644; AVX512-NEXT: br i1 [[CMP_NOT10]], label [[FOR_END:%.*]], label [[ITER_CHECK:%.*]]645; AVX512: iter.check:646; AVX512-NEXT: [[MUL:%.*]] = sub nsw i32 0, [[D]]647; AVX512-NEXT: [[IDXPROM:%.*]] = sext i32 [[MUL]] to i64648; AVX512-NEXT: [[TMP0:%.*]] = shl nsw i64 [[IDX_EXT]], 2649; AVX512-NEXT: [[TMP1:%.*]] = add nsw i64 [[TMP0]], -4650; AVX512-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 2651; AVX512-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1652; AVX512-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 8653; AVX512-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]654; AVX512: vector.memcheck:655; AVX512-NEXT: [[TMP4:%.*]] = shl nsw i64 [[IDX_EXT]], 2656; AVX512-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP4]], -4657; AVX512-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP5]], 2658; AVX512-NEXT: [[TMP7:%.*]] = shl i64 [[TMP6]], 6659; AVX512-NEXT: [[TMP8:%.*]] = add nuw nsw i64 [[TMP7]], 8660; AVX512-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DEST:%.*]], i64 [[TMP8]]661; AVX512-NEXT: [[TMP11:%.*]] = mul nsw i64 [[IDX_EXT]], -4662; AVX512-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP11]]663; AVX512-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DEST]], [[SCEVGEP1]]664; AVX512-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[PTR]], [[SCEVGEP]]665; AVX512-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]666; AVX512-NEXT: [[BOUND04:%.*]] = icmp ult ptr [[DEST]], [[PTR]]667; AVX512-NEXT: [[BOUND15:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP]]668; AVX512-NEXT: [[FOUND_CONFLICT6:%.*]] = and i1 [[BOUND04]], [[BOUND15]]669; AVX512-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]]670; AVX512-NEXT: br i1 [[CONFLICT_RDX]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]671; AVX512: vector.main.loop.iter.check:672; AVX512-NEXT: [[MIN_ITERS_CHECK7:%.*]] = icmp ult i64 [[TMP3]], 16673; AVX512-NEXT: br i1 [[MIN_ITERS_CHECK7]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]674; AVX512: vector.ph:675; AVX512-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 16676; AVX512-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]677; AVX512-NEXT: [[TMP13:%.*]] = mul i64 [[N_VEC]], 64678; AVX512-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[DEST]], i64 [[TMP13]]679; AVX512-NEXT: br label [[VECTOR_BODY:%.*]]680; AVX512: vector.body:681; AVX512-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]682; AVX512-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[DEST]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]683; AVX512-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <16 x i64> <i64 0, i64 64, i64 128, i64 192, i64 256, i64 320, i64 384, i64 448, i64 512, i64 576, i64 640, i64 704, i64 768, i64 832, i64 896, i64 960>684; AVX512-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 4685; AVX512-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[OFFSET_IDX]]686; AVX512-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[IDXPROM]]687; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <16 x float>, ptr [[TMP17]], align 4, !alias.scope [[META8:![0-9]+]]688; AVX512-NEXT: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> [[WIDE_LOAD]], <16 x ptr> align 4 [[TMP14]], <16 x i1> splat (i1 true)), !alias.scope [[META11:![0-9]+]], !noalias [[META13:![0-9]+]]689; AVX512-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x float>, ptr [[TMP16]], align 4, !alias.scope [[META15:![0-9]+]]690; AVX512-NEXT: [[TMP20:%.*]] = getelementptr inbounds float, <16 x ptr> [[TMP14]], i64 1691; AVX512-NEXT: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> [[WIDE_LOAD6]], <16 x ptr> align 4 [[TMP20]], <16 x i1> splat (i1 true)), !alias.scope [[META11]], !noalias [[META13]]692; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16693; AVX512-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 1024694; AVX512-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]695; AVX512-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]696; AVX512: middle.block:697; AVX512-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]698; AVX512-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]699; AVX512: vec.epilog.iter.check:700; AVX512-NEXT: [[TMP23:%.*]] = mul i64 [[N_VEC]], 4701; AVX512-NEXT: [[IND_END12:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP23]]702; AVX512-NEXT: [[TMP38:%.*]] = mul i64 [[N_VEC]], 64703; AVX512-NEXT: [[IND_END15:%.*]] = getelementptr i8, ptr [[DEST]], i64 [[TMP38]]704; AVX512-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8705; AVX512-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF17:![0-9]+]]706; AVX512: vec.epilog.ph:707; AVX512-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]708; AVX512-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[DEST]], [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]709; AVX512-NEXT: [[N_MOD_VF9:%.*]] = urem i64 [[TMP3]], 8710; AVX512-NEXT: [[N_VEC10:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF9]]711; AVX512-NEXT: [[TMP24:%.*]] = mul i64 [[N_VEC10]], 4712; AVX512-NEXT: [[IND_END11:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP24]]713; AVX512-NEXT: [[TMP25:%.*]] = mul i64 [[N_VEC10]], 64714; AVX512-NEXT: [[IND_END14:%.*]] = getelementptr i8, ptr [[DEST]], i64 [[TMP25]]715; AVX512-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]716; AVX512: vec.epilog.vector.body:717; AVX512-NEXT: [[INDEX18:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT24:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]718; AVX512-NEXT: [[POINTER_PHI19:%.*]] = phi ptr [ [[BC_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[PTR_IND20:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]719; AVX512-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[POINTER_PHI19]], <8 x i64> <i64 0, i64 64, i64 128, i64 192, i64 256, i64 320, i64 384, i64 448>720; AVX512-NEXT: [[OFFSET_IDX21:%.*]] = mul i64 [[INDEX18]], 4721; AVX512-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[OFFSET_IDX21]]722; AVX512-NEXT: [[TMP29:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i64 [[IDXPROM]]723; AVX512-NEXT: [[WIDE_LOAD13:%.*]] = load <8 x float>, ptr [[TMP29]], align 4, !alias.scope [[META8]]724; AVX512-NEXT: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> [[WIDE_LOAD13]], <8 x ptr> align 4 [[TMP26]], <8 x i1> splat (i1 true)), !alias.scope [[META11]], !noalias [[META13]]725; AVX512-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x float>, ptr [[TMP28]], align 4, !alias.scope [[META15]]726; AVX512-NEXT: [[TMP32:%.*]] = getelementptr inbounds float, <8 x ptr> [[TMP26]], i64 1727; AVX512-NEXT: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> [[WIDE_LOAD14]], <8 x ptr> align 4 [[TMP32]], <8 x i1> splat (i1 true)), !alias.scope [[META11]], !noalias [[META13]]728; AVX512-NEXT: [[INDEX_NEXT24]] = add nuw i64 [[INDEX18]], 8729; AVX512-NEXT: [[PTR_IND20]] = getelementptr i8, ptr [[POINTER_PHI19]], i64 512730; AVX512-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT24]], [[N_VEC10]]731; AVX512-NEXT: br i1 [[TMP33]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]732; AVX512: vec.epilog.middle.block:733; AVX512-NEXT: [[CMP_N17:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC10]]734; AVX512-NEXT: br i1 [[CMP_N17]], label [[FOR_END_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]735; AVX512: vec.epilog.scalar.ph:736;737; FVW2-LABEL: @test_gather_not_profitable_pr48429(738; FVW2-NEXT: entry:739; FVW2-NEXT: [[IDX_EXT:%.*]] = sext i32 [[D:%.*]] to i64740; FVW2-NEXT: [[SCEVGEP1:%.*]] = getelementptr float, ptr [[PTR:%.*]], i64 [[IDX_EXT]]741; FVW2-NEXT: [[CMP_NOT10:%.*]] = icmp eq i32 [[D]], 0742; FVW2-NEXT: br i1 [[CMP_NOT10]], label [[FOR_END:%.*]], label [[FOR_BODY_LR_PH:%.*]]743; FVW2: for.body.lr.ph:744; FVW2-NEXT: [[MUL:%.*]] = sub nsw i32 0, [[D]]745; FVW2-NEXT: [[IDXPROM:%.*]] = sext i32 [[MUL]] to i64746; FVW2-NEXT: [[TMP0:%.*]] = shl nsw i64 [[IDX_EXT]], 2747; FVW2-NEXT: [[TMP1:%.*]] = add nsw i64 [[TMP0]], -4748; FVW2-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 2749; FVW2-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1750; FVW2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2751; FVW2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]752; FVW2: vector.memcheck:753; FVW2-NEXT: [[TMP4:%.*]] = shl nsw i64 [[IDX_EXT]], 2754; FVW2-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP4]], -4755; FVW2-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP5]], 2756; FVW2-NEXT: [[TMP7:%.*]] = shl i64 [[TMP6]], 6757; FVW2-NEXT: [[TMP8:%.*]] = add nuw nsw i64 [[TMP7]], 8758; FVW2-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DEST:%.*]], i64 [[TMP8]]759; FVW2-NEXT: [[TMP11:%.*]] = mul nsw i64 [[IDX_EXT]], -4760; FVW2-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP11]]761; FVW2-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DEST]], [[SCEVGEP1]]762; FVW2-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[PTR]], [[SCEVGEP]]763; FVW2-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]764; FVW2-NEXT: [[BOUND04:%.*]] = icmp ult ptr [[DEST]], [[PTR]]765; FVW2-NEXT: [[BOUND15:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP]]766; FVW2-NEXT: [[FOUND_CONFLICT6:%.*]] = and i1 [[BOUND04]], [[BOUND15]]767; FVW2-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]]768; FVW2-NEXT: br i1 [[CONFLICT_RDX]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]769; FVW2: vector.ph:770; FVW2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2771; FVW2-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]772; FVW2-NEXT: [[TMP13:%.*]] = mul i64 [[N_VEC]], 4773; FVW2-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP13]]774; FVW2-NEXT: [[TMP14:%.*]] = mul i64 [[N_VEC]], 64775; FVW2-NEXT: [[IND_END7:%.*]] = getelementptr i8, ptr [[DEST]], i64 [[TMP14]]776; FVW2-NEXT: br label [[VECTOR_BODY:%.*]]777; FVW2: vector.body:778; FVW2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]779; FVW2-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 4780; FVW2-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[OFFSET_IDX]]781; FVW2-NEXT: [[OFFSET_IDX9:%.*]] = mul i64 [[INDEX]], 64782; FVW2-NEXT: [[TMP17:%.*]] = add i64 [[OFFSET_IDX9]], 0783; FVW2-NEXT: [[TMP18:%.*]] = add i64 [[OFFSET_IDX9]], 64784; FVW2-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[DEST]], i64 [[TMP17]]785; FVW2-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[DEST]], i64 [[TMP18]]786; FVW2-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[IDXPROM]]787; FVW2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP21]], align 4, !alias.scope [[META8:![0-9]+]]788; FVW2-NEXT: [[TMP23:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i32 0789; FVW2-NEXT: [[TMP24:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i32 1790; FVW2-NEXT: store float [[TMP23]], ptr [[TMP19]], align 4, !alias.scope [[META11:![0-9]+]], !noalias [[META13:![0-9]+]]791; FVW2-NEXT: store float [[TMP24]], ptr [[TMP20]], align 4, !alias.scope [[META11]], !noalias [[META13]]792; FVW2-NEXT: [[WIDE_LOAD10:%.*]] = load <2 x float>, ptr [[TMP16]], align 4, !alias.scope [[META15:![0-9]+]]793; FVW2-NEXT: [[TMP28:%.*]] = extractelement <2 x float> [[WIDE_LOAD10]], i32 0794; FVW2-NEXT: [[TMP29:%.*]] = extractelement <2 x float> [[WIDE_LOAD10]], i32 1795; FVW2-NEXT: [[TMP25:%.*]] = getelementptr inbounds float, ptr [[TMP19]], i64 1796; FVW2-NEXT: [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP20]], i64 1797; FVW2-NEXT: store float [[TMP28]], ptr [[TMP25]], align 4, !alias.scope [[META11]], !noalias [[META13]]798; FVW2-NEXT: store float [[TMP29]], ptr [[TMP22]], align 4, !alias.scope [[META11]], !noalias [[META13]]799; FVW2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2800; FVW2-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]801; FVW2-NEXT: br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]802; FVW2: middle.block:803; FVW2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]804; FVW2-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]805; FVW2: scalar.ph:806;807entry:808 %idx.ext = sext i32 %d to i64809 %add.ptr = getelementptr inbounds float, ptr %ptr, i64 %idx.ext810 %cmp.not10 = icmp eq i32 %d, 0811 br i1 %cmp.not10, label %for.end, label %for.body.lr.ph812 813for.body.lr.ph:814 %mul = sub nsw i32 0, %d815 %idxprom = sext i32 %mul to i64816 br label %for.body817 818for.body:819 %ptr.addr.012 = phi ptr [ %ptr, %for.body.lr.ph ], [ %incdec.ptr, %for.body ]820 %dest.addr.011 = phi ptr [ %dest, %for.body.lr.ph ], [ %add.ptr6, %for.body ]821 %arrayidx = getelementptr inbounds float, ptr %ptr.addr.012, i64 %idxprom822 %0 = load float, ptr %arrayidx, align 4823 store float %0, ptr %dest.addr.011, align 4824 %1 = load float, ptr %ptr.addr.012, align 4825 %arrayidx5 = getelementptr inbounds float, ptr %dest.addr.011, i64 1826 store float %1, ptr %arrayidx5, align 4827 %incdec.ptr = getelementptr inbounds float, ptr %ptr.addr.012, i64 1828 %add.ptr6 = getelementptr inbounds float, ptr %dest.addr.011, i64 16829 %cmp.not = icmp eq ptr %incdec.ptr, %add.ptr830 br i1 %cmp.not, label %for.end, label %for.body831 832for.end:833 ret void834}835