460 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --replace-value-regex "!llvm.loop ![0-9]+"2; RUN: opt -passes='loop-vectorize' -force-vector-width=2 -enable-epilogue-vectorization -epilogue-vectorization-force-VF=2 -S %s | FileCheck %s3 4target triple = "arm64-apple-macosx"5 6; Test for #57712.7define void @test_widen_ptr_induction(ptr %ptr.start.1) {8; CHECK-LABEL: @test_widen_ptr_induction(9; CHECK-NEXT: iter.check:10; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]11; CHECK: vector.main.loop.iter.check:12; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]13; CHECK: vector.ph:14; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[PTR_START_1:%.*]], i64 1000015; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]16; CHECK: vector.body:17; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]18; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[PTR_START_1]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]19; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <2 x i64> <i64 0, i64 1>20; CHECK-NEXT: [[NEXT_GEP:%.*]] = extractelement <2 x ptr> [[TMP5]], i32 021; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, <2 x ptr> [[TMP5]], <2 x i64> splat (i64 2)22; CHECK-NEXT: [[TMP8:%.*]] = icmp ne <2 x ptr> [[TMP5]], zeroinitializer23; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP8]], i32 024; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i1> [[TMP8]], i32 125; CHECK-NEXT: [[TMP9:%.*]] = icmp ne <2 x ptr> [[TMP7]], zeroinitializer26; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i1> [[TMP9]], i32 027; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[TMP9]], i32 128; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP11]])29; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP12]])30; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP13]])31; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP14]])32; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 233; CHECK-NEXT: store <2 x i8> zeroinitializer, ptr [[NEXT_GEP]], align 134; CHECK-NEXT: store <2 x i8> zeroinitializer, ptr [[TMP15]], align 135; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 436; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 437; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000038; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}39; CHECK: middle.block:40; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]41; CHECK: vec.epilog.iter.check:42; CHECK-NEXT: [[IND_END4:%.*]] = getelementptr i8, ptr [[PTR_START_1]], i64 1000043; CHECK-NEXT: br i1 true, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]44; CHECK: vec.epilog.ph:45; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]46; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[TMP0]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PTR_START_1]], [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]47; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[PTR_START_1]], i64 1000048; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]49; CHECK: vec.epilog.vector.body:50; CHECK-NEXT: [[INDEX6:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]51; CHECK-NEXT: [[POINTER_PHI2:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], [[VEC_EPILOG_PH]] ], [ [[PTR_IND5:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]52; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[POINTER_PHI2]], <2 x i64> <i64 0, i64 1>53; CHECK-NEXT: [[NEXT_GEP7:%.*]] = extractelement <2 x ptr> [[TMP20]], i32 054; CHECK-NEXT: [[TMP21:%.*]] = icmp ne <2 x ptr> [[TMP20]], zeroinitializer55; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x i1> [[TMP21]], i32 056; CHECK-NEXT: [[TMP23:%.*]] = extractelement <2 x i1> [[TMP21]], i32 157; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP22]])58; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP23]])59; CHECK-NEXT: store <2 x i8> zeroinitializer, ptr [[NEXT_GEP7]], align 160; CHECK-NEXT: [[INDEX_NEXT9]] = add nuw i64 [[INDEX6]], 261; CHECK-NEXT: [[PTR_IND5]] = getelementptr i8, ptr [[POINTER_PHI2]], i64 262; CHECK-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT9]], 1000063; CHECK-NEXT: br i1 [[TMP25]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], {{!llvm.loop ![0-9]+}}64; CHECK: vec.epilog.middle.block:65; CHECK-NEXT: br i1 false, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]66; CHECK: vec.epilog.scalar.ph:67; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 10000, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]68; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PTR_START_1]], [[ITER_CHECK]] ]69; CHECK-NEXT: br label [[LOOP:%.*]]70; CHECK: loop:71; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]72; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL5]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], [[LOOP]] ]73; CHECK-NEXT: [[CMP_I_I_I_I:%.*]] = icmp ne ptr [[PTR_IV]], null74; CHECK-NEXT: tail call void @llvm.assume(i1 [[CMP_I_I_I_I]])75; CHECK-NEXT: store i8 0, ptr [[PTR_IV]], align 176; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 177; CHECK-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds i8, ptr [[PTR_IV]], i64 178; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 1000079; CHECK-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP]], {{!llvm.loop ![0-9]+}}80; CHECK: exit:81; CHECK-NEXT: ret void82;83entry:84 br label %loop85 86loop:87 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]88 %ptr.iv = phi ptr [ %ptr.start.1, %entry ], [ %ptr.iv.next, %loop ]89 %cmp.i.i.i.i = icmp ne ptr %ptr.iv, null90 tail call void @llvm.assume(i1 %cmp.i.i.i.i)91 store i8 0, ptr %ptr.iv, align 192 %iv.next = add nuw nsw i64 %iv, 193 %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 194 %ec = icmp eq i64 %iv, 1000095 br i1 %ec, label %exit, label %loop96 97exit:98 ret void99}100 101declare void @llvm.assume(i1 noundef)102 103define void @test_widen_induction(ptr %A, i64 %N) {104; CHECK-LABEL: @test_widen_induction(105; CHECK-NEXT: iter.check:106; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 2107; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]108; CHECK: vector.main.loop.iter.check:109; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 4110; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]111; CHECK: vector.ph:112; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4113; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]114; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]115; CHECK: vector.body:116; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]117; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]118; CHECK-NEXT: [[STEP_ADD:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 2)119; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]120; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i64 2121; CHECK-NEXT: store <2 x i64> [[VEC_IND]], ptr [[TMP1]], align 4122; CHECK-NEXT: store <2 x i64> [[STEP_ADD]], ptr [[TMP3]], align 4123; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4124; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD]], splat (i64 2)125; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]126; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}127; CHECK: middle.block:128; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]129; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]130; CHECK: vec.epilog.iter.check:131; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 2132; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3]]133; CHECK: vec.epilog.ph:134; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]135; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 2136; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]137; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0138; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i64> [[DOTSPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer139; CHECK-NEXT: [[INDUCTION:%.*]] = add nuw nsw <2 x i64> [[DOTSPLAT]], <i64 0, i64 1>140; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]141; CHECK: vec.epilog.vector.body:142; CHECK-NEXT: [[INDEX5:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]143; CHECK-NEXT: [[VEC_IND6:%.*]] = phi <2 x i64> [ [[INDUCTION]], [[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT7:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]144; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX5]]145; CHECK-NEXT: store <2 x i64> [[VEC_IND6]], ptr [[TMP6]], align 4146; CHECK-NEXT: [[INDEX_NEXT8]] = add nuw i64 [[INDEX5]], 2147; CHECK-NEXT: [[VEC_IND_NEXT7]] = add nuw nsw <2 x i64> [[VEC_IND6]], splat (i64 2)148; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT8]], [[N_VEC3]]149; CHECK-NEXT: br i1 [[TMP8]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], {{!llvm.loop ![0-9]+}}150; CHECK: vec.epilog.middle.block:151; CHECK-NEXT: [[CMP_N9:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]152; CHECK-NEXT: br i1 [[CMP_N9]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]153; CHECK: vec.epilog.scalar.ph:154; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]155; CHECK-NEXT: br label [[LOOP:%.*]]156; CHECK: loop:157; CHECK-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL4]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ]158; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV_1]]159; CHECK-NEXT: store i64 [[IV_1]], ptr [[GEP_A]], align 4160; CHECK-NEXT: [[IV_1_NEXT]] = add nuw nsw i64 [[IV_1]], 1161; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_1_NEXT]], [[N]]162; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], {{!llvm.loop ![0-9]+}}163; CHECK: exit:164; CHECK-NEXT: ret void165;166entry:167 br label %loop168 169loop:170 %iv.1 = phi i64 [ 0, %entry ], [ %iv.1.next, %loop ]171 %gep.A = getelementptr inbounds i64, ptr %A, i64 %iv.1172 store i64 %iv.1, ptr %gep.A, align 4173 %iv.1.next = add nuw nsw i64 %iv.1, 1174 %exitcond = icmp eq i64 %iv.1.next, %N175 br i1 %exitcond, label %exit, label %loop176 177exit:178 ret void179}180 181define void @test_widen_induction_variable_start(ptr %A, i64 %N, i64 %start) {182; CHECK-LABEL: @test_widen_induction_variable_start(183; CHECK-NEXT: iter.check:184; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[N:%.*]], [[START:%.*]]185; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2186; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]187; CHECK: vector.main.loop.iter.check:188; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 4189; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]190; CHECK: vector.ph:191; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4192; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]193; CHECK-NEXT: [[IND_END:%.*]] = add i64 [[START]], [[N_VEC]]194; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[START]], i64 0195; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i64> [[DOTSPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer196; CHECK-NEXT: [[INDUCTION:%.*]] = add nuw nsw <2 x i64> [[DOTSPLAT]], <i64 0, i64 1>197; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]198; CHECK: vector.body:199; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]200; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]201; CHECK-NEXT: [[STEP_ADD:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 2)202; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[START]], [[INDEX]]203; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[OFFSET_IDX]]204; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i64 2205; CHECK-NEXT: store <2 x i64> [[VEC_IND]], ptr [[TMP2]], align 4206; CHECK-NEXT: store <2 x i64> [[STEP_ADD]], ptr [[TMP4]], align 4207; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4208; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD]], splat (i64 2)209; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]210; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}211; CHECK: middle.block:212; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]213; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]214; CHECK: vec.epilog.iter.check:215; CHECK-NEXT: [[IND_END5:%.*]] = add i64 [[START]], [[N_VEC]]216; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 2217; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3]]218; CHECK: vec.epilog.ph:219; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]220; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]221; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[TMP0]], 2222; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF2]]223; CHECK-NEXT: [[IND_END4:%.*]] = add i64 [[START]], [[N_VEC3]]224; CHECK-NEXT: [[DOTSPLATINSERT8:%.*]] = insertelement <2 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0225; CHECK-NEXT: [[DOTSPLAT9:%.*]] = shufflevector <2 x i64> [[DOTSPLATINSERT8]], <2 x i64> poison, <2 x i32> zeroinitializer226; CHECK-NEXT: [[INDUCTION10:%.*]] = add nuw nsw <2 x i64> [[DOTSPLAT9]], <i64 0, i64 1>227; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]228; CHECK: vec.epilog.vector.body:229; CHECK-NEXT: [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT14:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]230; CHECK-NEXT: [[VEC_IND11:%.*]] = phi <2 x i64> [ [[INDUCTION10]], [[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT12:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]231; CHECK-NEXT: [[OFFSET_IDX13:%.*]] = add i64 [[START]], [[INDEX7]]232; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX13]]233; CHECK-NEXT: store <2 x i64> [[VEC_IND11]], ptr [[TMP7]], align 4234; CHECK-NEXT: [[INDEX_NEXT14]] = add nuw i64 [[INDEX7]], 2235; CHECK-NEXT: [[VEC_IND_NEXT12]] = add nuw nsw <2 x i64> [[VEC_IND11]], splat (i64 2)236; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT14]], [[N_VEC3]]237; CHECK-NEXT: br i1 [[TMP9]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], {{!llvm.loop ![0-9]+}}238; CHECK: vec.epilog.middle.block:239; CHECK-NEXT: [[CMP_N15:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC3]]240; CHECK-NEXT: br i1 [[CMP_N15]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]241; CHECK: vec.epilog.scalar.ph:242; CHECK-NEXT: [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[IND_END4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END5]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK:%.*]] ]243; CHECK-NEXT: br label [[LOOP:%.*]]244; CHECK: loop:245; CHECK-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL6]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ]246; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV_1]]247; CHECK-NEXT: store i64 [[IV_1]], ptr [[GEP_A]], align 4248; CHECK-NEXT: [[IV_1_NEXT]] = add nuw nsw i64 [[IV_1]], 1249; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_1_NEXT]], [[N]]250; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], {{!llvm.loop ![0-9]+}}251; CHECK: exit:252; CHECK-NEXT: ret void253;254entry:255 br label %loop256 257loop:258 %iv.1 = phi i64 [ %start, %entry ], [ %iv.1.next, %loop ]259 %gep.A = getelementptr inbounds i64, ptr %A, i64 %iv.1260 store i64 %iv.1, ptr %gep.A, align 4261 %iv.1.next = add nuw nsw i64 %iv.1, 1262 %exitcond = icmp eq i64 %iv.1.next, %N263 br i1 %exitcond, label %exit, label %loop264 265exit:266 ret void267}268 269define void @test_widen_induction_step_2(ptr %A, i64 %N, i32 %step) {270; CHECK-LABEL: @test_widen_induction_step_2(271; CHECK-NEXT: iter.check:272; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 2273; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]274; CHECK: vector.main.loop.iter.check:275; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 4276; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]277; CHECK: vector.ph:278; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4279; CHECK-NEXT: [[IND_END4:%.*]] = sub i64 [[N]], [[N_MOD_VF]]280; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]281; CHECK: vector.body:282; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]283; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]284; CHECK-NEXT: [[STEP_ADD:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 2)285; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]286; CHECK-NEXT: [[TMP2:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 10)287; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i64> [[STEP_ADD]], splat (i64 10)288; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i64 2289; CHECK-NEXT: store <2 x i64> [[TMP2]], ptr [[TMP1]], align 4290; CHECK-NEXT: store <2 x i64> [[TMP3]], ptr [[TMP5]], align 4291; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4292; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[STEP_ADD]], splat (i64 2)293; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[IND_END4]]294; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}295; CHECK: middle.block:296; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[IND_END4]]297; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]298; CHECK: vec.epilog.iter.check:299; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 2300; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3]]301; CHECK: vec.epilog.ph:302; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]303; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 2304; CHECK-NEXT: [[IND_END:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]305; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0306; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i64> [[DOTSPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer307; CHECK-NEXT: [[INDUCTION:%.*]] = add nuw nsw <2 x i64> [[DOTSPLAT]], <i64 0, i64 1>308; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]309; CHECK: vec.epilog.vector.body:310; CHECK-NEXT: [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT10:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]311; CHECK-NEXT: [[VEC_IND8:%.*]] = phi <2 x i64> [ [[INDUCTION]], [[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT9:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]312; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX7]]313; CHECK-NEXT: [[TMP9:%.*]] = add <2 x i64> [[VEC_IND8]], splat (i64 10)314; CHECK-NEXT: store <2 x i64> [[TMP9]], ptr [[TMP8]], align 4315; CHECK-NEXT: [[INDEX_NEXT10]] = add nuw i64 [[INDEX7]], 2316; CHECK-NEXT: [[VEC_IND_NEXT9]] = add nuw nsw <2 x i64> [[VEC_IND8]], splat (i64 2)317; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT10]], [[IND_END]]318; CHECK-NEXT: br i1 [[TMP11]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], {{!llvm.loop ![0-9]+}}319; CHECK: vec.epilog.middle.block:320; CHECK-NEXT: [[CMP_N11:%.*]] = icmp eq i64 [[N]], [[IND_END]]321; CHECK-NEXT: br i1 [[CMP_N11]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]322; CHECK: vec.epilog.scalar.ph:323; CHECK-NEXT: [[BC_RESUME_VAL6:%.*]] = phi i64 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]324; CHECK-NEXT: br label [[LOOP:%.*]]325; CHECK: loop:326; CHECK-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL6]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP]] ]327; CHECK-NEXT: [[IV_2:%.*]] = phi i64 [ [[BC_RESUME_VAL6]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_1_NEXT]], [[LOOP]] ]328; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV_1]]329; CHECK-NEXT: [[ADD:%.*]] = add i64 [[IV_2]], 10330; CHECK-NEXT: store i64 [[ADD]], ptr [[GEP_A]], align 4331; CHECK-NEXT: [[IV_1_NEXT]] = add nuw nsw i64 [[IV_1]], 1332; CHECK-NEXT: [[IV_2_NEXT:%.*]] = add nuw nsw i64 [[IV_1]], 2333; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_1_NEXT]], [[N]]334; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], {{!llvm.loop ![0-9]+}}335; CHECK: exit:336; CHECK-NEXT: ret void337;338entry:339 br label %loop340 341loop:342 %iv.1 = phi i64 [ 0, %entry ], [ %iv.1.next, %loop ]343 %iv.2 = phi i64 [ 0, %entry ], [ %iv.1.next, %loop ]344 %gep.A = getelementptr inbounds i64, ptr %A, i64 %iv.1345 %add = add i64 %iv.2, 10346 store i64 %add, ptr %gep.A, align 4347 %iv.1.next = add nuw nsw i64 %iv.1, 1348 %iv.2.next = add nuw nsw i64 %iv.1, 2349 %exitcond = icmp eq i64 %iv.1.next, %N350 br i1 %exitcond, label %exit, label %loop351 352exit:353 ret void354}355 356define void @test_widen_extended_induction(ptr %dst) {357; CHECK-LABEL: @test_widen_extended_induction(358; CHECK-NEXT: entry:359; CHECK-NEXT: br label [[LOOP:%.*]]360; CHECK: loop:361; CHECK-NEXT: [[IV:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]362; CHECK-NEXT: [[IV_EXT:%.*]] = zext i8 [[IV]] to i64363; CHECK-NEXT: [[ARRAYIDX1449:%.*]] = getelementptr inbounds [6 x i8], ptr [[DST:%.*]], i64 0, i64 [[IV_EXT]]364; CHECK-NEXT: store i8 [[IV]], ptr [[ARRAYIDX1449]], align 1365; CHECK-NEXT: [[IV_NEXT]] = add i8 [[IV]], 1366; CHECK-NEXT: [[IV_NEXT_EXT:%.*]] = zext i8 [[IV_NEXT]] to i32367; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT_EXT]], 10000368; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]]369; CHECK: exit:370; CHECK-NEXT: ret void371;372entry:373 br label %loop374 375loop:376 %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]377 %iv.ext = zext i8 %iv to i64378 %arrayidx1449 = getelementptr inbounds [6 x i8], ptr %dst, i64 0, i64 %iv.ext379 store i8 %iv, ptr %arrayidx1449, align 1380 %iv.next = add i8 %iv, 1381 %iv.next.ext = zext i8 %iv.next to i32382 %ec = icmp eq i32 %iv.next.ext, 10000383 br i1 %ec, label %exit, label %loop384 385exit:386 ret void387}388 389define void @test_widen_truncated_induction(ptr %A) {390; CHECK-LABEL: @test_widen_truncated_induction(391; CHECK-NEXT: iter.check:392; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]393; CHECK: vector.main.loop.iter.check:394; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]395; CHECK: vector.ph:396; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]397; CHECK: vector.body:398; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]399; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i8> [ <i8 0, i8 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]400; CHECK-NEXT: [[STEP_ADD:%.*]] = add <2 x i8> [[VEC_IND]], splat (i8 2)401; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]402; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 2403; CHECK-NEXT: store <2 x i8> [[VEC_IND]], ptr [[TMP1]], align 1404; CHECK-NEXT: store <2 x i8> [[STEP_ADD]], ptr [[TMP3]], align 1405; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4406; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i8> [[STEP_ADD]], splat (i8 2)407; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000408; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], {{!llvm.loop ![0-9]+}}409; CHECK: middle.block:410; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]411; CHECK: vec.epilog.iter.check:412; CHECK-NEXT: br i1 true, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3]]413; CHECK: vec.epilog.ph:414; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]415; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[VEC_EPILOG_RESUME_VAL]] to i8416; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i8> poison, i8 [[TMP5]], i64 0417; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i8> [[DOTSPLATINSERT]], <2 x i8> poison, <2 x i32> zeroinitializer418; CHECK-NEXT: [[INDUCTION:%.*]] = add <2 x i8> [[DOTSPLAT]], <i8 0, i8 1>419; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]420; CHECK: vec.epilog.vector.body:421; CHECK-NEXT: [[INDEX2:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]422; CHECK-NEXT: [[VEC_IND3:%.*]] = phi <2 x i8> [ [[INDUCTION]], [[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT4:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]423; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX2]]424; CHECK-NEXT: store <2 x i8> [[VEC_IND3]], ptr [[TMP7]], align 1425; CHECK-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX2]], 2426; CHECK-NEXT: [[VEC_IND_NEXT4]] = add <2 x i8> [[VEC_IND3]], splat (i8 2)427; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT5]], 10000428; CHECK-NEXT: br i1 [[TMP9]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], {{!llvm.loop ![0-9]+}}429; CHECK: vec.epilog.middle.block:430; CHECK-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]431; CHECK: vec.epilog.scalar.ph:432; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 10000, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]433; CHECK-NEXT: br label [[LOOP:%.*]]434; CHECK: loop:435; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]436; CHECK-NEXT: [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i8437; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]438; CHECK-NEXT: store i8 [[IV_TRUNC]], ptr [[ARRAYIDX]], align 1439; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1440; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[IV_NEXT]], 10000441; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT]], {{!llvm.loop ![0-9]+}}442; CHECK: exit:443; CHECK-NEXT: ret void444;445entry:446 br label %loop447 448loop:449 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]450 %iv.trunc = trunc i64 %iv to i8451 %arrayidx = getelementptr inbounds i8, ptr %A, i64 %iv452 store i8 %iv.trunc, ptr %arrayidx, align 1453 %iv.next = add nuw nsw i64 %iv, 1454 %exitcond = icmp ne i64 %iv.next, 10000455 br i1 %exitcond, label %loop, label %exit456 457exit:458 ret void459}460