228 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=4 -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -pass-remarks='loop-vectorize' -disable-output -S 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS3; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=4 -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S | FileCheck %s4 5; These tests are to check that fold-tail procedure produces correct scalar code when6; loop-vectorization is only unrolling but not vectorizing.7 8; CHECK-REMARKS: remark: {{.*}} interleaved loop (interleaved count: 4)9; CHECK-REMARKS-NEXT: remark: {{.*}} interleaved loop (interleaved count: 4)10; CHECK-REMARKS-NOT: remark: {{.*}} vectorized loop11 12define void @VF1-VPlanExe(ptr %dst) {13; CHECK-LABEL: @VF1-VPlanExe(14; CHECK-NEXT: entry:15; CHECK-NEXT: br label [[VECTOR_PH:%.*]]16; CHECK: vector.ph:17; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]18; CHECK: vector.body:19; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]20; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 021; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 122; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 223; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 324; CHECK-NEXT: [[TMP4:%.*]] = icmp ule i64 [[TMP0]], 1425; CHECK-NEXT: [[TMP5:%.*]] = icmp ule i64 [[TMP1]], 1426; CHECK-NEXT: [[TMP6:%.*]] = icmp ule i64 [[TMP2]], 1427; CHECK-NEXT: [[TMP7:%.*]] = icmp ule i64 [[TMP3]], 1428; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]29; CHECK: pred.store.if:30; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[TMP0]]31; CHECK-NEXT: store i32 0, ptr [[TMP8]], align 432; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]33; CHECK: pred.store.continue:34; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]35; CHECK: pred.store.if1:36; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP1]]37; CHECK-NEXT: store i32 0, ptr [[TMP9]], align 438; CHECK-NEXT: br label [[PRED_STORE_CONTINUE2]]39; CHECK: pred.store.continue2:40; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]41; CHECK: pred.store.if3:42; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP2]]43; CHECK-NEXT: store i32 0, ptr [[TMP10]], align 444; CHECK-NEXT: br label [[PRED_STORE_CONTINUE4]]45; CHECK: pred.store.continue4:46; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]47; CHECK: pred.store.if5:48; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP3]]49; CHECK-NEXT: store i32 0, ptr [[TMP11]], align 450; CHECK-NEXT: br label [[PRED_STORE_CONTINUE6]]51; CHECK: pred.store.continue6:52; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 453; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1654; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]55; CHECK: middle.block:56; CHECK-NEXT: br label [[FOR_BODY:%.*]]57; CHECK: for.cond.cleanup:58; CHECK-NEXT: ret void59;60entry:61 br label %for.body62 63for.cond.cleanup:64 ret void65 66for.body:67 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]68 %dst.ptr = getelementptr inbounds i32, ptr %dst, i64 %indvars.iv69 store i32 0, ptr %dst.ptr70 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 171 %exitcond = icmp eq i64 %indvars.iv.next, 1572 br i1 %exitcond, label %for.cond.cleanup, label %for.body73}74 75; Note: Most scalar pointer induction GEPs could be sunk into the conditional76; blocks.77define void @VF1-VPWidenCanonicalIVRecipeExe(ptr %ptr1) {78; CHECK-LABEL: @VF1-VPWidenCanonicalIVRecipeExe(79; CHECK-NEXT: entry:80; CHECK-NEXT: [[PTR2:%.*]] = getelementptr inbounds double, ptr [[PTR1:%.*]], i64 1581; CHECK-NEXT: br label [[VECTOR_PH:%.*]]82; CHECK: vector.ph:83; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]84; CHECK: vector.body:85; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE12:%.*]] ]86; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 887; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 888; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 1689; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 2490; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[OFFSET_IDX]]91; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP4]]92; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP5]]93; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP6]]94; CHECK-NEXT: [[VEC_IV:%.*]] = add i64 [[INDEX]], 095; CHECK-NEXT: [[VEC_IV4:%.*]] = add i64 [[INDEX]], 196; CHECK-NEXT: [[VEC_IV5:%.*]] = add i64 [[INDEX]], 297; CHECK-NEXT: [[VEC_IV6:%.*]] = add i64 [[INDEX]], 398; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i64 [[VEC_IV]], 1499; CHECK-NEXT: [[TMP1:%.*]] = icmp ule i64 [[VEC_IV4]], 14100; CHECK-NEXT: [[TMP2:%.*]] = icmp ule i64 [[VEC_IV5]], 14101; CHECK-NEXT: [[TMP3:%.*]] = icmp ule i64 [[VEC_IV6]], 14102; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]103; CHECK: pred.store.if:104; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP]], align 8105; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]106; CHECK: pred.store.continue:107; CHECK-NEXT: br i1 [[TMP1]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]108; CHECK: pred.store.if7:109; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP1]], align 8110; CHECK-NEXT: br label [[PRED_STORE_CONTINUE8]]111; CHECK: pred.store.continue8:112; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]113; CHECK: pred.store.if9:114; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP2]], align 8115; CHECK-NEXT: br label [[PRED_STORE_CONTINUE10]]116; CHECK: pred.store.continue10:117; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12]]118; CHECK: pred.store.if11:119; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP3]], align 8120; CHECK-NEXT: br label [[PRED_STORE_CONTINUE12]]121; CHECK: pred.store.continue12:122; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4123; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16124; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]125; CHECK: middle.block:126; CHECK-NEXT: br label [[FOR_BODY:%.*]]127; CHECK: for.cond.cleanup:128; CHECK-NEXT: ret void129;130entry:131 %ptr2 = getelementptr inbounds double, ptr %ptr1, i64 15132 br label %for.body133 134for.cond.cleanup:135 ret void136 137for.body:138 %addr = phi ptr [ %ptr, %for.body ], [ %ptr1, %entry ]139 store double 0.0, ptr %addr140 %ptr = getelementptr inbounds double, ptr %addr, i64 1141 %cond = icmp eq ptr %ptr, %ptr2142 br i1 %cond, label %for.cond.cleanup, label %for.body143}144 145define i64 @live_out_scalar_vf(i64 %n) {146; CHECK-LABEL: @live_out_scalar_vf(147; CHECK-NEXT: entry:148; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1149; CHECK-NEXT: br label [[VECTOR_PH:%.*]]150; CHECK: vector.ph:151; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], 15152; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 16153; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]154; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1155; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0156; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer157; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]158; CHECK: vector.body:159; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]160; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]161; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, [[VECTOR_PH]] ], [ [[STEP_ADD_3:%.*]], [[VECTOR_BODY]] ]162; CHECK-NEXT: [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)163; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i64> [[STEP_ADD]], splat (i64 4)164; CHECK-NEXT: [[STEP_ADD_3]] = add <4 x i64> [[STEP_ADD_2]], splat (i64 4)165; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]166; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt <4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]167; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt <4 x i64> [[STEP_ADD_2]], [[BROADCAST_SPLAT]]168; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt <4 x i64> [[STEP_ADD_3]], [[BROADCAST_SPLAT]]169; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 16170; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD_3]], splat (i64 4)171; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]172; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]173; CHECK: middle.block:174; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)175; CHECK-NEXT: [[TMP6:%.*]] = add i64 12, [[FIRST_INACTIVE_LANE]]176; CHECK-NEXT: [[FIRST_INACTIVE_LANE1:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP3]], i1 false)177; CHECK-NEXT: [[TMP7:%.*]] = add i64 8, [[FIRST_INACTIVE_LANE1]]178; CHECK-NEXT: [[TMP8:%.*]] = icmp ne i64 [[FIRST_INACTIVE_LANE1]], 4179; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 [[TMP6]]180; CHECK-NEXT: [[FIRST_INACTIVE_LANE2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)181; CHECK-NEXT: [[TMP10:%.*]] = add i64 4, [[FIRST_INACTIVE_LANE2]]182; CHECK-NEXT: [[TMP11:%.*]] = icmp ne i64 [[FIRST_INACTIVE_LANE2]], 4183; CHECK-NEXT: [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[TMP10]], i64 [[TMP9]]184; CHECK-NEXT: [[FIRST_INACTIVE_LANE3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP1]], i1 false)185; CHECK-NEXT: [[TMP13:%.*]] = add i64 0, [[FIRST_INACTIVE_LANE3]]186; CHECK-NEXT: [[TMP14:%.*]] = icmp ne i64 [[FIRST_INACTIVE_LANE3]], 4187; CHECK-NEXT: [[TMP15:%.*]] = select i1 [[TMP14]], i64 [[TMP13]], i64 [[TMP12]]188; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[TMP15]], 1189; CHECK-NEXT: [[TMP16:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], 1190; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[TMP16]]191; CHECK-NEXT: [[TMP18:%.*]] = sub i64 [[TMP16]], 4192; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i64> [[STEP_ADD]], i64 [[TMP18]]193; CHECK-NEXT: [[TMP20:%.*]] = icmp uge i64 [[TMP16]], 4194; CHECK-NEXT: [[TMP21:%.*]] = select i1 [[TMP20]], i64 [[TMP32]], i64 [[TMP17]]195; CHECK-NEXT: [[TMP22:%.*]] = sub i64 [[TMP16]], 8196; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i64> [[STEP_ADD_2]], i64 [[TMP22]]197; CHECK-NEXT: [[TMP24:%.*]] = icmp uge i64 [[TMP16]], 8198; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP24]], i64 [[TMP23]], i64 [[TMP21]]199; CHECK-NEXT: [[TMP26:%.*]] = sub i64 [[TMP16]], 12200; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[STEP_ADD_3]], i64 [[TMP26]]201; CHECK-NEXT: [[TMP28:%.*]] = icmp uge i64 [[TMP16]], 12202; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP25]]203; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i64> [[VECTOR_RECUR]], i32 3204; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[LAST_ACTIVE_LANE]], 0205; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP31]], i64 [[TMP30]], i64 [[TMP29]]206; CHECK-NEXT: br label [[LOOP:%.*]]207; CHECK: exit:208; CHECK-NEXT: ret i64 [[TMP19]]209;210entry:211 br label %loop212 213loop:214 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]215 ; Need to use a phi otherwise the header mask will use a216 ; VPWidenCanonicalIVRecipe instead of a VPScalarIVStepsRecipe.217 %exitval = phi i64 [ 0, %entry ], [ %iv, %loop ]218 %iv.next = add i64 %iv, 1219 %ec = icmp eq i64 %iv, %n220 br i1 %ec, label %exit, label %loop221 222exit:223 ret i64 %exitval224}225 226;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:227; CHECK-REMARKS: {{.*}}228