526 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -passes=loop-vectorize -force-vector-width=4 -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -tail-predication=force-enabled -S %s -o - | FileCheck %s3 4target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"5 6define void @test_stride1_4i32(ptr readonly %data, ptr noalias nocapture %dst, i32 %n) {7; CHECK-LABEL: @test_stride1_4i32(8; CHECK-NEXT: entry:9; CHECK-NEXT: br label [[VECTOR_PH:%.*]]10; CHECK: vector.ph:11; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[N:%.*]], 312; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 413; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]14; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]15; CHECK: vector.body:16; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]17; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]])18; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i32 [[INDEX]], 219; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DATA:%.*]], i32 [[TMP1]]20; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)21; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> splat (i32 5), [[WIDE_MASKED_LOAD]]22; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDEX]]23; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP4]], ptr align 4 [[TMP5]], <4 x i1> [[ACTIVE_LANE_MASK]])24; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 425; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]26; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]27; CHECK: middle.block:28; CHECK-NEXT: br label [[FOR_BODY:%.*]]29; CHECK: end:30; CHECK-NEXT: ret void31;32entry:33 br label %for.body34for.body: ; preds = %for.body.preheader, %for.body35 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]36 %mul = mul nuw nsw i32 %i.023, 137 %add5 = add nuw nsw i32 %mul, 238 %arrayidx6 = getelementptr inbounds i32, ptr %data, i32 %add539 %0 = load i32, ptr %arrayidx6, align 440 %add7 = add nsw i32 5, %041 %arrayidx9 = getelementptr inbounds i32, ptr %dst, i32 %i.02342 store i32 %add7, ptr %arrayidx9, align 443 %inc = add nuw nsw i32 %i.023, 144 %exitcond.not = icmp eq i32 %inc, %n45 br i1 %exitcond.not, label %end, label %for.body46end: ; preds = %end, %entry47 ret void48}49 50define void @test_stride-1_4i32(ptr readonly %data, ptr noalias nocapture %dst, i32 %n) {51; CHECK-LABEL: @test_stride-1_4i32(52; CHECK-NEXT: entry:53; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], 454; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]55; CHECK: vector.ph:56; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 457; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]58; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]59; CHECK: vector.body:60; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]61; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw i32 [[INDEX]], -162; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 263; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DATA:%.*]], i32 [[TMP2]]64; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i32 065; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i32 -366; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP5]], align 467; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>68; CHECK-NEXT: [[TMP6:%.*]] = add nsw <4 x i32> splat (i32 5), [[REVERSE]]69; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDEX]]70; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr [[TMP7]], align 471; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 472; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]73; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]74; CHECK: middle.block:75; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]76; CHECK-NEXT: br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]77; CHECK: scalar.ph:78; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]79; CHECK-NEXT: br label [[FOR_BODY:%.*]]80; CHECK: for.body:81; CHECK-NEXT: [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]82; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], -183; CHECK-NEXT: [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 284; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i32 [[ADD5]]85; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX6]], align 486; CHECK-NEXT: [[ADD7:%.*]] = add nsw i32 5, [[TMP10]]87; CHECK-NEXT: [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[I_023]]88; CHECK-NEXT: store i32 [[ADD7]], ptr [[ARRAYIDX9]], align 489; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_023]], 190; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]91; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[END]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]92; CHECK: end:93; CHECK-NEXT: ret void94;95entry:96 br label %for.body97for.body: ; preds = %for.body.preheader, %for.body98 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]99 %mul = mul nuw nsw i32 %i.023, -1100 %add5 = add nuw nsw i32 %mul, 2101 %arrayidx6 = getelementptr inbounds i32, ptr %data, i32 %add5102 %0 = load i32, ptr %arrayidx6, align 4103 %add7 = add nsw i32 5, %0104 %arrayidx9 = getelementptr inbounds i32, ptr %dst, i32 %i.023105 store i32 %add7, ptr %arrayidx9, align 4106 %inc = add nuw nsw i32 %i.023, 1107 %exitcond.not = icmp eq i32 %inc, %n108 br i1 %exitcond.not, label %end, label %for.body109end: ; preds = %end, %entry110 ret void111}112 113define void @test_stride2_4i32(ptr readonly %data, ptr noalias nocapture %dst, i32 %n) {114;115; CHECK-LABEL: @test_stride2_4i32(116; CHECK-NEXT: entry:117; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[N:%.*]], 4118; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]119; CHECK: vector.ph:120; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 4121; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[N_MOD_VF]], 0122; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[TMP0]], i32 4, i32 [[N_MOD_VF]]123; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[TMP1]]124; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]125; CHECK: vector.body:126; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]127; CHECK-NEXT: [[TMP3:%.*]] = mul nuw nsw i32 [[INDEX]], 2128; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i32 [[TMP3]], 2129; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DATA:%.*]], i32 [[TMP4]]130; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP5]], align 4131; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>132; CHECK-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> splat (i32 5), [[STRIDED_VEC]]133; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDEX]]134; CHECK-NEXT: store <4 x i32> [[TMP7]], ptr [[TMP8]], align 4135; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4136; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]137; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]138; CHECK: middle.block:139; CHECK-NEXT: br label [[SCALAR_PH]]140; CHECK: scalar.ph:141; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]142; CHECK-NEXT: br label [[FOR_BODY:%.*]]143; CHECK: for.body:144; CHECK-NEXT: [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]145; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], 2146; CHECK-NEXT: [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2147; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i32 [[ADD5]]148; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[ARRAYIDX6]], align 4149; CHECK-NEXT: [[ADD7:%.*]] = add nsw i32 5, [[TMP11]]150; CHECK-NEXT: [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[I_023]]151; CHECK-NEXT: store i32 [[ADD7]], ptr [[ARRAYIDX9]], align 4152; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_023]], 1153; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]154; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]155; CHECK: end:156; CHECK-NEXT: ret void157;158entry:159 br label %for.body160for.body: ; preds = %for.body.preheader, %for.body161 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]162 %mul = mul nuw nsw i32 %i.023, 2163 %add5 = add nuw nsw i32 %mul, 2164 %arrayidx6 = getelementptr inbounds i32, ptr %data, i32 %add5165 %0 = load i32, ptr %arrayidx6, align 4166 %add7 = add nsw i32 5, %0167 %arrayidx9 = getelementptr inbounds i32, ptr %dst, i32 %i.023168 store i32 %add7, ptr %arrayidx9, align 4169 %inc = add nuw nsw i32 %i.023, 1170 %exitcond.not = icmp eq i32 %inc, %n171 br i1 %exitcond.not, label %end, label %for.body172end: ; preds = %end, %entry173 ret void174}175 176define void @test_stride3_4i32(ptr readonly %data, ptr noalias nocapture %dst, i32 %n) {177; CHECK-LABEL: @test_stride3_4i32(178; CHECK-NEXT: entry:179; CHECK-NEXT: br label [[VECTOR_PH:%.*]]180; CHECK: vector.ph:181; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[N:%.*]], 3182; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 4183; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]184; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]185; CHECK: vector.body:186; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]187; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]188; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]])189; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw <4 x i32> [[VEC_IND]], splat (i32 3)190; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw <4 x i32> [[TMP1]], splat (i32 2)191; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DATA:%.*]], <4 x i32> [[TMP2]]192; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP3]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)193; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> splat (i32 5), [[WIDE_MASKED_GATHER]]194; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDEX]]195; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP4]], ptr align 4 [[TMP5]], <4 x i1> [[ACTIVE_LANE_MASK]])196; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4197; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)198; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]199; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]200; CHECK: middle.block:201; CHECK-NEXT: br label [[FOR_BODY:%.*]]202; CHECK: end:203; CHECK-NEXT: ret void204;205entry:206 br label %for.body207for.body: ; preds = %for.body.preheader, %for.body208 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]209 %mul = mul nuw nsw i32 %i.023, 3210 %add5 = add nuw nsw i32 %mul, 2211 %arrayidx6 = getelementptr inbounds i32, ptr %data, i32 %add5212 %0 = load i32, ptr %arrayidx6, align 4213 %add7 = add nsw i32 5, %0214 %arrayidx9 = getelementptr inbounds i32, ptr %dst, i32 %i.023215 store i32 %add7, ptr %arrayidx9, align 4216 %inc = add nuw nsw i32 %i.023, 1217 %exitcond.not = icmp eq i32 %inc, %n218 br i1 %exitcond.not, label %end, label %for.body219end: ; preds = %end, %entry220 ret void221}222 223define void @test_stride4_4i32(ptr readonly %data, ptr noalias nocapture %dst, i32 %n) {224; CHECK-LABEL: @test_stride4_4i32(225; CHECK-NEXT: entry:226; CHECK-NEXT: br label [[VECTOR_PH:%.*]]227; CHECK: vector.ph:228; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[N:%.*]], 3229; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 4230; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]231; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]232; CHECK: vector.body:233; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]234; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]235; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]])236; CHECK-NEXT: [[TMP1:%.*]] = mul nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)237; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw <4 x i32> [[TMP1]], splat (i32 2)238; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DATA:%.*]], <4 x i32> [[TMP2]]239; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP3]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)240; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> splat (i32 5), [[WIDE_MASKED_GATHER]]241; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDEX]]242; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP4]], ptr align 4 [[TMP5]], <4 x i1> [[ACTIVE_LANE_MASK]])243; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4244; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)245; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]246; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]247; CHECK: middle.block:248; CHECK-NEXT: br label [[FOR_BODY:%.*]]249; CHECK: end:250; CHECK-NEXT: ret void251;252entry:253 br label %for.body254for.body: ; preds = %for.body.preheader, %for.body255 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]256 %mul = mul nuw nsw i32 %i.023, 4257 %add5 = add nuw nsw i32 %mul, 2258 %arrayidx6 = getelementptr inbounds i32, ptr %data, i32 %add5259 %0 = load i32, ptr %arrayidx6, align 4260 %add7 = add nsw i32 5, %0261 %arrayidx9 = getelementptr inbounds i32, ptr %dst, i32 %i.023262 store i32 %add7, ptr %arrayidx9, align 4263 %inc = add nuw nsw i32 %i.023, 1264 %exitcond.not = icmp eq i32 %inc, %n265 br i1 %exitcond.not, label %end, label %for.body266end: ; preds = %end, %entry267 ret void268}269 270define void @test_stride_loopinvar_4i32(ptr readonly %data, ptr noalias nocapture %dst, i32 %n, i32 %stride) {271; CHECK-LABEL: @test_stride_loopinvar_4i32(272; CHECK-NEXT: entry:273; CHECK-NEXT: br label [[VECTOR_SCEVCHECK:%.*]]274; CHECK: vector.scevcheck:275; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE:%.*]], 1276; CHECK-NEXT: br i1 [[IDENT_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]277; CHECK: vector.ph:278; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[N:%.*]], 3279; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 4280; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]281; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]282; CHECK: vector.body:283; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]284; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]])285; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i32 [[INDEX]], 2286; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DATA:%.*]], i32 [[TMP1]]287; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)288; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> splat (i32 5), [[WIDE_MASKED_LOAD]]289; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDEX]]290; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP4]], ptr align 4 [[TMP5]], <4 x i1> [[ACTIVE_LANE_MASK]])291; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4292; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]293; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]294; CHECK: middle.block:295; CHECK-NEXT: br label [[END:%.*]]296; CHECK: scalar.ph:297; CHECK-NEXT: br label [[FOR_BODY:%.*]]298; CHECK: for.body:299; CHECK-NEXT: [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]300; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], [[STRIDE]]301; CHECK-NEXT: [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2302; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i32 [[ADD5]]303; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX6]], align 4304; CHECK-NEXT: [[ADD7:%.*]] = add nsw i32 5, [[TMP8]]305; CHECK-NEXT: [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[I_023]]306; CHECK-NEXT: store i32 [[ADD7]], ptr [[ARRAYIDX9]], align 4307; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_023]], 1308; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]309; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[END]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]310; CHECK: end:311; CHECK-NEXT: ret void312;313entry:314 br label %for.body315for.body: ; preds = %for.body.preheader, %for.body316 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]317 %mul = mul nuw nsw i32 %i.023, %stride318 %add5 = add nuw nsw i32 %mul, 2319 %arrayidx6 = getelementptr inbounds i32, ptr %data, i32 %add5320 %0 = load i32, ptr %arrayidx6, align 4321 %add7 = add nsw i32 5, %0322 %arrayidx9 = getelementptr inbounds i32, ptr %dst, i32 %i.023323 store i32 %add7, ptr %arrayidx9, align 4324 %inc = add nuw nsw i32 %i.023, 1325 %exitcond.not = icmp eq i32 %inc, %n326 br i1 %exitcond.not, label %end, label %for.body327end: ; preds = %end, %entry328 ret void329}330 331define void @test_stride_noninvar_4i32(ptr readonly %data, ptr noalias nocapture %dst, i32 %n) {332; CHECK-LABEL: @test_stride_noninvar_4i32(333; CHECK-NEXT: entry:334; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], 4335; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]336; CHECK: vector.ph:337; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 4338; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]339; CHECK-NEXT: [[TMP0:%.*]] = mul i32 [[N_VEC]], 8340; CHECK-NEXT: [[IND_END:%.*]] = add i32 3, [[TMP0]]341; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]342; CHECK: vector.body:343; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]344; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]345; CHECK-NEXT: [[VEC_IND2:%.*]] = phi <4 x i32> [ <i32 3, i32 11, i32 19, i32 27>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]346; CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw <4 x i32> [[VEC_IND]], [[VEC_IND2]]347; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw <4 x i32> [[TMP2]], splat (i32 2)348; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[DATA:%.*]], <4 x i32> [[TMP3]]349; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP4]], <4 x i1> splat (i1 true), <4 x i32> poison)350; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> splat (i32 5), [[WIDE_MASKED_GATHER]]351; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDEX]]352; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP6]], align 4353; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4354; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)355; CHECK-NEXT: [[VEC_IND_NEXT3]] = add nuw nsw <4 x i32> [[VEC_IND2]], splat (i32 32)356; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]357; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]358; CHECK: middle.block:359; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]360; CHECK-NEXT: br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]361; CHECK: scalar.ph:362; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]363; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY]] ]364; CHECK-NEXT: br label [[FOR_BODY:%.*]]365; CHECK: for.body:366; CHECK-NEXT: [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]367; CHECK-NEXT: [[STRIDE:%.*]] = phi i32 [ [[NEXT_STRIDE:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]368; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], [[STRIDE]]369; CHECK-NEXT: [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2370; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i32 [[ADD5]]371; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX6]], align 4372; CHECK-NEXT: [[ADD7:%.*]] = add nsw i32 5, [[TMP9]]373; CHECK-NEXT: [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[I_023]]374; CHECK-NEXT: store i32 [[ADD7]], ptr [[ARRAYIDX9]], align 4375; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_023]], 1376; CHECK-NEXT: [[NEXT_STRIDE]] = add nuw nsw i32 [[STRIDE]], 8377; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]378; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[END]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]379; CHECK: end:380; CHECK-NEXT: ret void381;382entry:383 br label %for.body384for.body: ; preds = %for.body.preheader, %for.body385 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]386 %stride = phi i32 [ %next.stride, %for.body ], [ 3, %entry ]387 %mul = mul nuw nsw i32 %i.023, %stride388 %add5 = add nuw nsw i32 %mul, 2389 %arrayidx6 = getelementptr inbounds i32, ptr %data, i32 %add5390 %0 = load i32, ptr %arrayidx6, align 4391 %add7 = add nsw i32 5, %0392 %arrayidx9 = getelementptr inbounds i32, ptr %dst, i32 %i.023393 store i32 %add7, ptr %arrayidx9, align 4394 %inc = add nuw nsw i32 %i.023, 1395 %next.stride = add nuw nsw i32 %stride, 8396 %exitcond.not = icmp eq i32 %inc, %n397 br i1 %exitcond.not, label %end, label %for.body398end: ; preds = %end, %entry399 ret void400}401 402define void @test_stride_noninvar2_4i32(ptr readonly %data, ptr noalias nocapture %dst, i32 %n) {403; CHECK-LABEL: @test_stride_noninvar2_4i32(404; CHECK-NEXT: entry:405; CHECK-NEXT: br label [[FOR_BODY:%.*]]406; CHECK: for.body:407; CHECK-NEXT: [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]408; CHECK-NEXT: [[STRIDE:%.*]] = phi i32 [ [[NEXT_STRIDE:%.*]], [[FOR_BODY]] ], [ 3, [[ENTRY]] ]409; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], [[STRIDE]]410; CHECK-NEXT: [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2411; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[DATA:%.*]], i32 [[ADD5]]412; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX6]], align 4413; CHECK-NEXT: [[ADD7:%.*]] = add nsw i32 5, [[TMP0]]414; CHECK-NEXT: [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[I_023]]415; CHECK-NEXT: store i32 [[ADD7]], ptr [[ARRAYIDX9]], align 4416; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_023]], 1417; CHECK-NEXT: [[NEXT_STRIDE]] = mul nuw nsw i32 [[STRIDE]], 8418; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N:%.*]]419; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[END:%.*]], label [[FOR_BODY]]420; CHECK: end:421; CHECK-NEXT: ret void422;423entry:424 br label %for.body425for.body: ; preds = %for.body.preheader, %for.body426 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]427 %stride = phi i32 [ %next.stride, %for.body ], [ 3, %entry ]428 %mul = mul nuw nsw i32 %i.023, %stride429 %add5 = add nuw nsw i32 %mul, 2430 %arrayidx6 = getelementptr inbounds i32, ptr %data, i32 %add5431 %0 = load i32, ptr %arrayidx6, align 4432 %add7 = add nsw i32 5, %0433 %arrayidx9 = getelementptr inbounds i32, ptr %dst, i32 %i.023434 store i32 %add7, ptr %arrayidx9, align 4435 %inc = add nuw nsw i32 %i.023, 1436 %next.stride = mul nuw nsw i32 %stride, 8437 %exitcond.not = icmp eq i32 %inc, %n438 br i1 %exitcond.not, label %end, label %for.body439end: ; preds = %end, %entry440 ret void441}442 443define void @test_stride_noninvar3_4i32(ptr readonly %data, ptr noalias nocapture %dst, i32 %n, i32 %x) {444; CHECK-LABEL: @test_stride_noninvar3_4i32(445; CHECK-NEXT: entry:446; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], 4447; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]448; CHECK: vector.ph:449; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 4450; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]451; CHECK-NEXT: [[TMP0:%.*]] = mul i32 [[N_VEC]], [[X:%.*]]452; CHECK-NEXT: [[IND_END:%.*]] = add i32 3, [[TMP0]]453; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0454; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer455; CHECK-NEXT: [[TMP3:%.*]] = mul nuw nsw <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[DOTSPLAT]]456; CHECK-NEXT: [[INDUCTION:%.*]] = add nuw nsw <4 x i32> splat (i32 3), [[TMP3]]457; CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i32 [[X]], 4458; CHECK-NEXT: [[DOTSPLATINSERT2:%.*]] = insertelement <4 x i32> poison, i32 [[TMP2]], i64 0459; CHECK-NEXT: [[DOTSPLAT3:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT2]], <4 x i32> poison, <4 x i32> zeroinitializer460; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]461; CHECK: vector.body:462; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]463; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]464; CHECK-NEXT: [[VEC_IND4:%.*]] = phi <4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ]465; CHECK-NEXT: [[TMP4:%.*]] = mul nuw nsw <4 x i32> [[VEC_IND]], [[VEC_IND4]]466; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw <4 x i32> [[TMP4]], splat (i32 2)467; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[DATA:%.*]], <4 x i32> [[TMP5]]468; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP6]], <4 x i1> splat (i1 true), <4 x i32> poison)469; CHECK-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> splat (i32 5), [[WIDE_MASKED_GATHER]]470; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDEX]]471; CHECK-NEXT: store <4 x i32> [[TMP7]], ptr [[TMP8]], align 4472; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4473; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)474; CHECK-NEXT: [[VEC_IND_NEXT5]] = add nuw nsw <4 x i32> [[VEC_IND4]], [[DOTSPLAT3]]475; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]476; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]477; CHECK: middle.block:478; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]479; CHECK-NEXT: br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]480; CHECK: scalar.ph:481; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]482; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY]] ]483; CHECK-NEXT: br label [[FOR_BODY:%.*]]484; CHECK: for.body:485; CHECK-NEXT: [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]486; CHECK-NEXT: [[STRIDE:%.*]] = phi i32 [ [[NEXT_STRIDE:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]487; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], [[STRIDE]]488; CHECK-NEXT: [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2489; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i32 [[ADD5]]490; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[ARRAYIDX6]], align 4491; CHECK-NEXT: [[ADD7:%.*]] = add nsw i32 5, [[TMP11]]492; CHECK-NEXT: [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[I_023]]493; CHECK-NEXT: store i32 [[ADD7]], ptr [[ARRAYIDX9]], align 4494; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_023]], 1495; CHECK-NEXT: [[NEXT_STRIDE]] = add nuw nsw i32 [[STRIDE]], [[X]]496; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]497; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[END]], label [[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]498; CHECK: end:499; CHECK-NEXT: ret void500;501entry:502 br label %for.body503for.body: ; preds = %for.body.preheader, %for.body504 %i.023 = phi i32 [ %inc, %for.body ], [ 0, %entry ]505 %stride = phi i32 [ %next.stride, %for.body ], [ 3, %entry ]506 %mul = mul nuw nsw i32 %i.023, %stride507 %add5 = add nuw nsw i32 %mul, 2508 %arrayidx6 = getelementptr inbounds i32, ptr %data, i32 %add5509 %0 = load i32, ptr %arrayidx6, align 4510 %add7 = add nsw i32 5, %0511 %arrayidx9 = getelementptr inbounds i32, ptr %dst, i32 %i.023512 store i32 %add7, ptr %arrayidx9, align 4513 %inc = add nuw nsw i32 %i.023, 1514 %next.stride = add nuw nsw i32 %stride, %x515 %exitcond.not = icmp eq i32 %inc, %n516 br i1 %exitcond.not, label %end, label %for.body517end: ; preds = %end, %entry518 ret void519}520 521declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)522declare <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x i32>)523declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)524declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32, <4 x i1>, <4 x i32>)525declare void @llvm.masked.scatter.v4i32.v4p0(<4 x i32>, <4 x ptr>, i32, <4 x i1>)526