1423 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -S -passes=loop-vectorize -force-vector-width=2 < %s | FileCheck %s3; RUN: opt -S -passes=loop-vectorize -force-vector-width=2 -prefer-predicate-over-epilogue=predicate-dont-vectorize < %s | FileCheck --check-prefix TAILFOLD %s4 5target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"6 7define void @bottom_tested(ptr %p, i32 %n) {8; CHECK-LABEL: @bottom_tested(9; CHECK-NEXT: entry:10; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 0)11; CHECK-NEXT: [[TMP0:%.*]] = add nuw i32 [[SMAX]], 112; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], 213; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]14; CHECK: vector.ph:15; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 216; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]]17; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]18; CHECK: vector.body:19; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]20; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[INDEX]] to i6421; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[TMP2]]22; CHECK-NEXT: store <2 x i16> zeroinitializer, ptr [[TMP3]], align 423; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 224; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]25; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]26; CHECK: middle.block:27; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]28; CHECK-NEXT: br i1 [[CMP_N]], label [[IF_END:%.*]], label [[SCALAR_PH]]29; CHECK: scalar.ph:30; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]31; CHECK-NEXT: br label [[FOR_COND:%.*]]32; CHECK: for.cond:33; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_COND]] ]34; CHECK-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i6435; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IPROM]]36; CHECK-NEXT: store i16 0, ptr [[B]], align 437; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 138; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]39; CHECK-NEXT: br i1 [[CMP]], label [[FOR_COND]], label [[IF_END]], !llvm.loop [[LOOP3:![0-9]+]]40; CHECK: if.end:41; CHECK-NEXT: ret void42;43; TAILFOLD-LABEL: @bottom_tested(44; TAILFOLD-NEXT: entry:45; TAILFOLD-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 0)46; TAILFOLD-NEXT: [[TMP0:%.*]] = add nuw i32 [[SMAX]], 147; TAILFOLD-NEXT: br label [[VECTOR_PH:%.*]]48; TAILFOLD: vector.ph:49; TAILFOLD-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP0]], 150; TAILFOLD-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 251; TAILFOLD-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]52; TAILFOLD-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i32 [[TMP0]], 153; TAILFOLD-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TRIP_COUNT_MINUS_1]], i64 054; TAILFOLD-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer55; TAILFOLD-NEXT: br label [[VECTOR_BODY:%.*]]56; TAILFOLD: vector.body:57; TAILFOLD-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]58; TAILFOLD-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE2]] ]59; TAILFOLD-NEXT: [[TMP1:%.*]] = icmp ule <2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]60; TAILFOLD-NEXT: [[TMP2:%.*]] = sext <2 x i32> [[VEC_IND]] to <2 x i64>61; TAILFOLD-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP1]], i32 062; TAILFOLD-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]63; TAILFOLD: pred.store.if:64; TAILFOLD-NEXT: [[TMP4:%.*]] = extractelement <2 x i64> [[TMP2]], i32 065; TAILFOLD-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[TMP4]]66; TAILFOLD-NEXT: store i16 0, ptr [[TMP5]], align 467; TAILFOLD-NEXT: br label [[PRED_STORE_CONTINUE]]68; TAILFOLD: pred.store.continue:69; TAILFOLD-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP1]], i32 170; TAILFOLD-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]71; TAILFOLD: pred.store.if1:72; TAILFOLD-NEXT: [[TMP7:%.*]] = extractelement <2 x i64> [[TMP2]], i32 173; TAILFOLD-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[TMP7]]74; TAILFOLD-NEXT: store i16 0, ptr [[TMP8]], align 475; TAILFOLD-NEXT: br label [[PRED_STORE_CONTINUE2]]76; TAILFOLD: pred.store.continue2:77; TAILFOLD-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 278; TAILFOLD-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)79; TAILFOLD-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]80; TAILFOLD-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]81; TAILFOLD: middle.block:82; TAILFOLD-NEXT: br label [[FOR_COND:%.*]]83; TAILFOLD: if.end:84; TAILFOLD-NEXT: ret void85;86entry:87 br label %for.cond88 89for.cond:90 %i = phi i32 [ 0, %entry ], [ %inc, %for.cond ]91 %iprom = sext i32 %i to i6492 %b = getelementptr inbounds i16, ptr %p, i64 %iprom93 store i16 0, ptr %b, align 494 %inc = add nsw i32 %i, 195 %cmp = icmp slt i32 %i, %n96 br i1 %cmp, label %for.cond, label %if.end97 98if.end:99 ret void100}101 102define void @early_exit(ptr %p, i32 %n) {103; CHECK-LABEL: @early_exit(104; CHECK-NEXT: entry:105; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 0)106; CHECK-NEXT: [[TMP0:%.*]] = add nuw i32 [[SMAX]], 1107; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], 2108; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]109; CHECK: vector.ph:110; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 2111; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[N_MOD_VF]], 0112; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 2, i32 [[N_MOD_VF]]113; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP2]]114; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]115; CHECK: vector.body:116; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]117; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[INDEX]] to i64118; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[TMP4]]119; CHECK-NEXT: store <2 x i16> zeroinitializer, ptr [[TMP5]], align 4120; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2121; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]122; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]123; CHECK: middle.block:124; CHECK-NEXT: br label [[SCALAR_PH]]125; CHECK: scalar.ph:126; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]127; CHECK-NEXT: br label [[FOR_COND:%.*]]128; CHECK: for.cond:129; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]130; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]131; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]132; CHECK: for.body:133; CHECK-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64134; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IPROM]]135; CHECK-NEXT: store i16 0, ptr [[B]], align 4136; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1137; CHECK-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]138; CHECK: if.end:139; CHECK-NEXT: ret void140;141; TAILFOLD-LABEL: @early_exit(142; TAILFOLD-NEXT: entry:143; TAILFOLD-NEXT: br label [[FOR_COND:%.*]]144; TAILFOLD: for.cond:145; TAILFOLD-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]146; TAILFOLD-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N:%.*]]147; TAILFOLD-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]148; TAILFOLD: for.body:149; TAILFOLD-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64150; TAILFOLD-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IPROM]]151; TAILFOLD-NEXT: store i16 0, ptr [[B]], align 4152; TAILFOLD-NEXT: [[INC]] = add nsw i32 [[I]], 1153; TAILFOLD-NEXT: br label [[FOR_COND]]154; TAILFOLD: if.end:155; TAILFOLD-NEXT: ret void156;157entry:158 br label %for.cond159 160for.cond:161 %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]162 %cmp = icmp slt i32 %i, %n163 br i1 %cmp, label %for.body, label %if.end164 165for.body:166 %iprom = sext i32 %i to i64167 %b = getelementptr inbounds i16, ptr %p, i64 %iprom168 store i16 0, ptr %b, align 4169 %inc = add nsw i32 %i, 1170 br label %for.cond171 172if.end:173 ret void174}175 176define i32 @early_exit_with_live_out(ptr %ptr) {177; CHECK-LABEL: @early_exit_with_live_out(178; CHECK-NEXT: entry:179; CHECK-NEXT: br label [[VECTOR_PH:%.*]]180; CHECK: vector.ph:181; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]182; CHECK: vector.body:183; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]184; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX]]185; CHECK-NEXT: store <2 x i32> splat (i32 10), ptr [[TMP1]], align 4186; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2187; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998188; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]189; CHECK: middle.block:190; CHECK-NEXT: br label [[SCALAR_PH:%.*]]191; CHECK: scalar.ph:192; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]193; CHECK: loop.header:194; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 998, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]195; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[IV]]196; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4197; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1198; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000199; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP_LATCH]]200; CHECK: loop.latch:201; CHECK-NEXT: store i32 10, ptr [[GEP]], align 4202; CHECK-NEXT: br label [[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]203; CHECK: exit:204; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i32 [ [[L]], [[LOOP_HEADER]] ]205; CHECK-NEXT: ret i32 [[RES_LCSSA]]206;207; TAILFOLD-LABEL: @early_exit_with_live_out(208; TAILFOLD-NEXT: entry:209; TAILFOLD-NEXT: br label [[LOOP_HEADER:%.*]]210; TAILFOLD: loop.header:211; TAILFOLD-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]212; TAILFOLD-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[IV]]213; TAILFOLD-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4214; TAILFOLD-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1215; TAILFOLD-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000216; TAILFOLD-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP_LATCH]]217; TAILFOLD: loop.latch:218; TAILFOLD-NEXT: store i32 10, ptr [[GEP]], align 4219; TAILFOLD-NEXT: br label [[LOOP_HEADER]]220; TAILFOLD: exit:221; TAILFOLD-NEXT: [[RES_LCSSA:%.*]] = phi i32 [ [[L]], [[LOOP_HEADER]] ]222; TAILFOLD-NEXT: ret i32 [[RES_LCSSA]]223;224entry:225 br label %loop.header226 227loop.header:228 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]229 %gep = getelementptr i32, ptr %ptr, i64 %iv230 %l = load i32, ptr %gep231 %iv.next = add nuw nsw i64 %iv, 1232 %ec = icmp eq i64 %iv.next, 1000233 br i1 %ec, label %exit, label %loop.latch234 235loop.latch:236 store i32 10, ptr %gep237 br label %loop.header238 239exit:240 %res.lcssa = phi i32 [ %l, %loop.header ]241 ret i32 %res.lcssa242}243 244; Same as early_exit, but with optsize to prevent the use of245; a scalar epilogue. -- Can't vectorize this in either case.246define void @optsize(ptr %p, i32 %n) optsize {247; CHECK-LABEL: @optsize(248; CHECK-NEXT: entry:249; CHECK-NEXT: br label [[FOR_COND:%.*]]250; CHECK: for.cond:251; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]252; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N:%.*]]253; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]254; CHECK: for.body:255; CHECK-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64256; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IPROM]]257; CHECK-NEXT: store i16 0, ptr [[B]], align 4258; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1259; CHECK-NEXT: br label [[FOR_COND]]260; CHECK: if.end:261; CHECK-NEXT: ret void262;263; TAILFOLD-LABEL: @optsize(264; TAILFOLD-NEXT: entry:265; TAILFOLD-NEXT: br label [[FOR_COND:%.*]]266; TAILFOLD: for.cond:267; TAILFOLD-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]268; TAILFOLD-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N:%.*]]269; TAILFOLD-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]270; TAILFOLD: for.body:271; TAILFOLD-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64272; TAILFOLD-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IPROM]]273; TAILFOLD-NEXT: store i16 0, ptr [[B]], align 4274; TAILFOLD-NEXT: [[INC]] = add nsw i32 [[I]], 1275; TAILFOLD-NEXT: br label [[FOR_COND]]276; TAILFOLD: if.end:277; TAILFOLD-NEXT: ret void278;279entry:280 br label %for.cond281 282for.cond:283 %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]284 %cmp = icmp slt i32 %i, %n285 br i1 %cmp, label %for.body, label %if.end286 287for.body:288 %iprom = sext i32 %i to i64289 %b = getelementptr inbounds i16, ptr %p, i64 %iprom290 store i16 0, ptr %b, align 4291 %inc = add nsw i32 %i, 1292 br label %for.cond293 294if.end:295 ret void296}297 298 299; multiple exit - no values inside the loop used outside300define void @multiple_unique_exit(ptr %p, i32 %n) {301; CHECK-LABEL: @multiple_unique_exit(302; CHECK-NEXT: entry:303; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 0)304; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[SMAX]], i32 2096)305; CHECK-NEXT: [[TMP0:%.*]] = add nuw nsw i32 [[UMIN]], 1306; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], 2307; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]308; CHECK: vector.ph:309; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 2310; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[N_MOD_VF]], 0311; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 2, i32 [[N_MOD_VF]]312; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP2]]313; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]314; CHECK: vector.body:315; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]316; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[INDEX]] to i64317; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[TMP4]]318; CHECK-NEXT: store <2 x i16> zeroinitializer, ptr [[TMP5]], align 4319; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2320; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]321; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]322; CHECK: middle.block:323; CHECK-NEXT: br label [[SCALAR_PH]]324; CHECK: scalar.ph:325; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]326; CHECK-NEXT: br label [[FOR_COND:%.*]]327; CHECK: for.cond:328; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]329; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]330; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]331; CHECK: for.body:332; CHECK-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64333; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IPROM]]334; CHECK-NEXT: store i16 0, ptr [[B]], align 4335; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1336; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i32 [[I]], 2096337; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END]], !llvm.loop [[LOOP9:![0-9]+]]338; CHECK: if.end:339; CHECK-NEXT: ret void340;341; TAILFOLD-LABEL: @multiple_unique_exit(342; TAILFOLD-NEXT: entry:343; TAILFOLD-NEXT: br label [[FOR_COND:%.*]]344; TAILFOLD: for.cond:345; TAILFOLD-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]346; TAILFOLD-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N:%.*]]347; TAILFOLD-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]348; TAILFOLD: for.body:349; TAILFOLD-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64350; TAILFOLD-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IPROM]]351; TAILFOLD-NEXT: store i16 0, ptr [[B]], align 4352; TAILFOLD-NEXT: [[INC]] = add nsw i32 [[I]], 1353; TAILFOLD-NEXT: [[CMP2:%.*]] = icmp slt i32 [[I]], 2096354; TAILFOLD-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END]]355; TAILFOLD: if.end:356; TAILFOLD-NEXT: ret void357;358entry:359 br label %for.cond360 361for.cond:362 %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]363 %cmp = icmp slt i32 %i, %n364 br i1 %cmp, label %for.body, label %if.end365 366for.body:367 %iprom = sext i32 %i to i64368 %b = getelementptr inbounds i16, ptr %p, i64 %iprom369 store i16 0, ptr %b, align 4370 %inc = add nsw i32 %i, 1371 %cmp2 = icmp slt i32 %i, 2096372 br i1 %cmp2, label %for.cond, label %if.end373 374if.end:375 ret void376}377 378; multiple exit - with an lcssa phi379define i32 @multiple_unique_exit2(ptr %p, i32 %n) {380; CHECK-LABEL: @multiple_unique_exit2(381; CHECK-NEXT: entry:382; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 0)383; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[SMAX]], i32 2096)384; CHECK-NEXT: [[TMP0:%.*]] = add nuw nsw i32 [[UMIN]], 1385; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], 2386; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]387; CHECK: vector.ph:388; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 2389; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[N_MOD_VF]], 0390; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 2, i32 [[N_MOD_VF]]391; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP2]]392; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]393; CHECK: vector.body:394; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]395; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[INDEX]] to i64396; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[TMP4]]397; CHECK-NEXT: store <2 x i16> zeroinitializer, ptr [[TMP5]], align 4398; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2399; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]400; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]401; CHECK: middle.block:402; CHECK-NEXT: br label [[SCALAR_PH]]403; CHECK: scalar.ph:404; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]405; CHECK-NEXT: br label [[FOR_COND:%.*]]406; CHECK: for.cond:407; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]408; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]409; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]410; CHECK: for.body:411; CHECK-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64412; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IPROM]]413; CHECK-NEXT: store i16 0, ptr [[B]], align 4414; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1415; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i32 [[I]], 2096416; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END]], !llvm.loop [[LOOP11:![0-9]+]]417; CHECK: if.end:418; CHECK-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[FOR_BODY]] ], [ [[I]], [[FOR_COND]] ]419; CHECK-NEXT: ret i32 [[I_LCSSA]]420;421; TAILFOLD-LABEL: @multiple_unique_exit2(422; TAILFOLD-NEXT: entry:423; TAILFOLD-NEXT: br label [[FOR_COND:%.*]]424; TAILFOLD: for.cond:425; TAILFOLD-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]426; TAILFOLD-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N:%.*]]427; TAILFOLD-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]428; TAILFOLD: for.body:429; TAILFOLD-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64430; TAILFOLD-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IPROM]]431; TAILFOLD-NEXT: store i16 0, ptr [[B]], align 4432; TAILFOLD-NEXT: [[INC]] = add nsw i32 [[I]], 1433; TAILFOLD-NEXT: [[CMP2:%.*]] = icmp slt i32 [[I]], 2096434; TAILFOLD-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END]]435; TAILFOLD: if.end:436; TAILFOLD-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[FOR_BODY]] ], [ [[I]], [[FOR_COND]] ]437; TAILFOLD-NEXT: ret i32 [[I_LCSSA]]438;439entry:440 br label %for.cond441 442for.cond:443 %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]444 %cmp = icmp slt i32 %i, %n445 br i1 %cmp, label %for.body, label %if.end446 447for.body:448 %iprom = sext i32 %i to i64449 %b = getelementptr inbounds i16, ptr %p, i64 %iprom450 store i16 0, ptr %b, align 4451 %inc = add nsw i32 %i, 1452 %cmp2 = icmp slt i32 %i, 2096453 br i1 %cmp2, label %for.cond, label %if.end454 455if.end:456 ret i32 %i457}458 459; multiple exit w/a non lcssa phi460define i32 @multiple_unique_exit3(ptr %p, i32 %n) {461; CHECK-LABEL: @multiple_unique_exit3(462; CHECK-NEXT: entry:463; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 0)464; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[SMAX]], i32 2096)465; CHECK-NEXT: [[TMP0:%.*]] = add nuw nsw i32 [[UMIN]], 1466; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], 2467; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]468; CHECK: vector.ph:469; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 2470; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[N_MOD_VF]], 0471; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 2, i32 [[N_MOD_VF]]472; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP2]]473; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]474; CHECK: vector.body:475; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]476; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[INDEX]] to i64477; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[TMP4]]478; CHECK-NEXT: store <2 x i16> zeroinitializer, ptr [[TMP5]], align 4479; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2480; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]481; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]482; CHECK: middle.block:483; CHECK-NEXT: br label [[SCALAR_PH]]484; CHECK: scalar.ph:485; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]486; CHECK-NEXT: br label [[FOR_COND:%.*]]487; CHECK: for.cond:488; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]489; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]490; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]491; CHECK: for.body:492; CHECK-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64493; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IPROM]]494; CHECK-NEXT: store i16 0, ptr [[B]], align 4495; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1496; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i32 [[I]], 2096497; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END]], !llvm.loop [[LOOP13:![0-9]+]]498; CHECK: if.end:499; CHECK-NEXT: [[EXIT:%.*]] = phi i32 [ 0, [[FOR_COND]] ], [ 1, [[FOR_BODY]] ]500; CHECK-NEXT: ret i32 [[EXIT]]501;502; TAILFOLD-LABEL: @multiple_unique_exit3(503; TAILFOLD-NEXT: entry:504; TAILFOLD-NEXT: br label [[FOR_COND:%.*]]505; TAILFOLD: for.cond:506; TAILFOLD-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]507; TAILFOLD-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N:%.*]]508; TAILFOLD-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]509; TAILFOLD: for.body:510; TAILFOLD-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64511; TAILFOLD-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IPROM]]512; TAILFOLD-NEXT: store i16 0, ptr [[B]], align 4513; TAILFOLD-NEXT: [[INC]] = add nsw i32 [[I]], 1514; TAILFOLD-NEXT: [[CMP2:%.*]] = icmp slt i32 [[I]], 2096515; TAILFOLD-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END]]516; TAILFOLD: if.end:517; TAILFOLD-NEXT: [[EXIT:%.*]] = phi i32 [ 0, [[FOR_COND]] ], [ 1, [[FOR_BODY]] ]518; TAILFOLD-NEXT: ret i32 [[EXIT]]519;520entry:521 br label %for.cond522 523for.cond:524 %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]525 %cmp = icmp slt i32 %i, %n526 br i1 %cmp, label %for.body, label %if.end527 528for.body:529 %iprom = sext i32 %i to i64530 %b = getelementptr inbounds i16, ptr %p, i64 %iprom531 store i16 0, ptr %b, align 4532 %inc = add nsw i32 %i, 1533 %cmp2 = icmp slt i32 %i, 2096534 br i1 %cmp2, label %for.cond, label %if.end535 536if.end:537 %exit = phi i32 [0, %for.cond], [1, %for.body]538 ret i32 %exit539}540 541; multiple exits w/distinct target blocks542define i32 @multiple_exit_blocks(ptr %p, i32 %n) {543; CHECK-LABEL: @multiple_exit_blocks(544; CHECK-NEXT: entry:545; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 0)546; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[SMAX]], i32 2096)547; CHECK-NEXT: [[TMP0:%.*]] = add nuw nsw i32 [[UMIN]], 1548; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], 2549; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]550; CHECK: vector.ph:551; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 2552; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[N_MOD_VF]], 0553; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 2, i32 [[N_MOD_VF]]554; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP2]]555; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]556; CHECK: vector.body:557; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]558; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[INDEX]] to i64559; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[TMP4]]560; CHECK-NEXT: store <2 x i16> zeroinitializer, ptr [[TMP5]], align 4561; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2562; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]563; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]564; CHECK: middle.block:565; CHECK-NEXT: br label [[SCALAR_PH]]566; CHECK: scalar.ph:567; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]568; CHECK-NEXT: br label [[FOR_COND:%.*]]569; CHECK: for.cond:570; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]571; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]572; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]573; CHECK: for.body:574; CHECK-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64575; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IPROM]]576; CHECK-NEXT: store i16 0, ptr [[B]], align 4577; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1578; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i32 [[I]], 2096579; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END2:%.*]], !llvm.loop [[LOOP15:![0-9]+]]580; CHECK: if.end:581; CHECK-NEXT: ret i32 0582; CHECK: if.end2:583; CHECK-NEXT: ret i32 1584;585; TAILFOLD-LABEL: @multiple_exit_blocks(586; TAILFOLD-NEXT: entry:587; TAILFOLD-NEXT: br label [[FOR_COND:%.*]]588; TAILFOLD: for.cond:589; TAILFOLD-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]590; TAILFOLD-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N:%.*]]591; TAILFOLD-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]592; TAILFOLD: for.body:593; TAILFOLD-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64594; TAILFOLD-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IPROM]]595; TAILFOLD-NEXT: store i16 0, ptr [[B]], align 4596; TAILFOLD-NEXT: [[INC]] = add nsw i32 [[I]], 1597; TAILFOLD-NEXT: [[CMP2:%.*]] = icmp slt i32 [[I]], 2096598; TAILFOLD-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END2:%.*]]599; TAILFOLD: if.end:600; TAILFOLD-NEXT: ret i32 0601; TAILFOLD: if.end2:602; TAILFOLD-NEXT: ret i32 1603;604entry:605 br label %for.cond606 607for.cond:608 %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]609 %cmp = icmp slt i32 %i, %n610 br i1 %cmp, label %for.body, label %if.end611 612for.body:613 %iprom = sext i32 %i to i64614 %b = getelementptr inbounds i16, ptr %p, i64 %iprom615 store i16 0, ptr %b, align 4616 %inc = add nsw i32 %i, 1617 %cmp2 = icmp slt i32 %i, 2096618 br i1 %cmp2, label %for.cond, label %if.end2619 620if.end:621 ret i32 0622 623if.end2:624 ret i32 1625}626 627; LCSSA, common value each exit628define i32 @multiple_exit_blocks2(ptr %p, i32 %n) {629; CHECK-LABEL: @multiple_exit_blocks2(630; CHECK-NEXT: entry:631; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 0)632; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[SMAX]], i32 2096)633; CHECK-NEXT: [[TMP0:%.*]] = add nuw nsw i32 [[UMIN]], 1634; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], 2635; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]636; CHECK: vector.ph:637; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 2638; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[N_MOD_VF]], 0639; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 2, i32 [[N_MOD_VF]]640; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP2]]641; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]642; CHECK: vector.body:643; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]644; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[INDEX]] to i64645; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[TMP4]]646; CHECK-NEXT: store <2 x i16> zeroinitializer, ptr [[TMP5]], align 4647; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2648; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]649; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]650; CHECK: middle.block:651; CHECK-NEXT: br label [[SCALAR_PH]]652; CHECK: scalar.ph:653; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]654; CHECK-NEXT: br label [[FOR_COND:%.*]]655; CHECK: for.cond:656; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]657; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]658; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]659; CHECK: for.body:660; CHECK-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64661; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IPROM]]662; CHECK-NEXT: store i16 0, ptr [[B]], align 4663; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1664; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i32 [[I]], 2096665; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END2:%.*]], !llvm.loop [[LOOP17:![0-9]+]]666; CHECK: if.end:667; CHECK-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[FOR_COND]] ]668; CHECK-NEXT: ret i32 [[I_LCSSA]]669; CHECK: if.end2:670; CHECK-NEXT: [[I_LCSSA1:%.*]] = phi i32 [ [[I]], [[FOR_BODY]] ]671; CHECK-NEXT: ret i32 [[I_LCSSA1]]672;673; TAILFOLD-LABEL: @multiple_exit_blocks2(674; TAILFOLD-NEXT: entry:675; TAILFOLD-NEXT: br label [[FOR_COND:%.*]]676; TAILFOLD: for.cond:677; TAILFOLD-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]678; TAILFOLD-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N:%.*]]679; TAILFOLD-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]680; TAILFOLD: for.body:681; TAILFOLD-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64682; TAILFOLD-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IPROM]]683; TAILFOLD-NEXT: store i16 0, ptr [[B]], align 4684; TAILFOLD-NEXT: [[INC]] = add nsw i32 [[I]], 1685; TAILFOLD-NEXT: [[CMP2:%.*]] = icmp slt i32 [[I]], 2096686; TAILFOLD-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END2:%.*]]687; TAILFOLD: if.end:688; TAILFOLD-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[FOR_COND]] ]689; TAILFOLD-NEXT: ret i32 [[I_LCSSA]]690; TAILFOLD: if.end2:691; TAILFOLD-NEXT: [[I_LCSSA1:%.*]] = phi i32 [ [[I]], [[FOR_BODY]] ]692; TAILFOLD-NEXT: ret i32 [[I_LCSSA1]]693;694entry:695 br label %for.cond696 697for.cond:698 %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]699 %cmp = icmp slt i32 %i, %n700 br i1 %cmp, label %for.body, label %if.end701 702for.body:703 %iprom = sext i32 %i to i64704 %b = getelementptr inbounds i16, ptr %p, i64 %iprom705 store i16 0, ptr %b, align 4706 %inc = add nsw i32 %i, 1707 %cmp2 = icmp slt i32 %i, 2096708 br i1 %cmp2, label %for.cond, label %if.end2709 710if.end:711 ret i32 %i712 713if.end2:714 ret i32 %i715}716 717; LCSSA, distinct value each exit718define i32 @multiple_exit_blocks3(ptr %p, i32 %n) {719; CHECK-LABEL: @multiple_exit_blocks3(720; CHECK-NEXT: entry:721; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 0)722; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[SMAX]], i32 2096)723; CHECK-NEXT: [[TMP0:%.*]] = add nuw nsw i32 [[UMIN]], 1724; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], 2725; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]726; CHECK: vector.ph:727; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 2728; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[N_MOD_VF]], 0729; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 2, i32 [[N_MOD_VF]]730; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP2]]731; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]732; CHECK: vector.body:733; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]734; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[INDEX]] to i64735; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[TMP4]]736; CHECK-NEXT: store <2 x i16> zeroinitializer, ptr [[TMP5]], align 4737; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2738; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]739; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]740; CHECK: middle.block:741; CHECK-NEXT: br label [[SCALAR_PH]]742; CHECK: scalar.ph:743; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]744; CHECK-NEXT: br label [[FOR_COND:%.*]]745; CHECK: for.cond:746; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]747; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]748; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]749; CHECK: for.body:750; CHECK-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64751; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IPROM]]752; CHECK-NEXT: store i16 0, ptr [[B]], align 4753; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1754; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i32 [[I]], 2096755; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END2:%.*]], !llvm.loop [[LOOP19:![0-9]+]]756; CHECK: if.end:757; CHECK-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[FOR_COND]] ]758; CHECK-NEXT: ret i32 [[I_LCSSA]]759; CHECK: if.end2:760; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], [[FOR_BODY]] ]761; CHECK-NEXT: ret i32 [[INC_LCSSA]]762;763; TAILFOLD-LABEL: @multiple_exit_blocks3(764; TAILFOLD-NEXT: entry:765; TAILFOLD-NEXT: br label [[FOR_COND:%.*]]766; TAILFOLD: for.cond:767; TAILFOLD-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]768; TAILFOLD-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N:%.*]]769; TAILFOLD-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]770; TAILFOLD: for.body:771; TAILFOLD-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64772; TAILFOLD-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IPROM]]773; TAILFOLD-NEXT: store i16 0, ptr [[B]], align 4774; TAILFOLD-NEXT: [[INC]] = add nsw i32 [[I]], 1775; TAILFOLD-NEXT: [[CMP2:%.*]] = icmp slt i32 [[I]], 2096776; TAILFOLD-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END2:%.*]]777; TAILFOLD: if.end:778; TAILFOLD-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[FOR_COND]] ]779; TAILFOLD-NEXT: ret i32 [[I_LCSSA]]780; TAILFOLD: if.end2:781; TAILFOLD-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], [[FOR_BODY]] ]782; TAILFOLD-NEXT: ret i32 [[INC_LCSSA]]783;784entry:785 br label %for.cond786 787for.cond:788 %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]789 %cmp = icmp slt i32 %i, %n790 br i1 %cmp, label %for.body, label %if.end791 792for.body:793 %iprom = sext i32 %i to i64794 %b = getelementptr inbounds i16, ptr %p, i64 %iprom795 store i16 0, ptr %b, align 4796 %inc = add nsw i32 %i, 1797 %cmp2 = icmp slt i32 %i, 2096798 br i1 %cmp2, label %for.cond, label %if.end2799 800if.end:801 ret i32 %i802 803if.end2:804 ret i32 %inc805}806 807; unique exit case but with a switch as two edges between the same pair of808; blocks is an often missed edge case809define i32 @multiple_exit_switch(ptr %p, i32 %n) {810; CHECK-LABEL: @multiple_exit_switch(811; CHECK-NEXT: entry:812; CHECK-NEXT: br label [[FOR_COND:%.*]]813; CHECK: for.cond:814; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND]] ]815; CHECK-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64816; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IPROM]]817; CHECK-NEXT: store i16 0, ptr [[B]], align 4818; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1819; CHECK-NEXT: switch i32 [[I]], label [[FOR_COND]] [820; CHECK-NEXT: i32 2096, label [[IF_END:%.*]]821; CHECK-NEXT: i32 2097, label [[IF_END]]822; CHECK-NEXT: ]823; CHECK: if.end:824; CHECK-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[FOR_COND]] ], [ [[I]], [[FOR_COND]] ]825; CHECK-NEXT: ret i32 [[I_LCSSA]]826;827; TAILFOLD-LABEL: @multiple_exit_switch(828; TAILFOLD-NEXT: entry:829; TAILFOLD-NEXT: br label [[FOR_COND:%.*]]830; TAILFOLD: for.cond:831; TAILFOLD-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND]] ]832; TAILFOLD-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64833; TAILFOLD-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IPROM]]834; TAILFOLD-NEXT: store i16 0, ptr [[B]], align 4835; TAILFOLD-NEXT: [[INC]] = add nsw i32 [[I]], 1836; TAILFOLD-NEXT: switch i32 [[I]], label [[FOR_COND]] [837; TAILFOLD-NEXT: i32 2096, label [[IF_END:%.*]]838; TAILFOLD-NEXT: i32 2097, label [[IF_END]]839; TAILFOLD-NEXT: ]840; TAILFOLD: if.end:841; TAILFOLD-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[FOR_COND]] ], [ [[I]], [[FOR_COND]] ]842; TAILFOLD-NEXT: ret i32 [[I_LCSSA]]843;844entry:845 br label %for.cond846 847for.cond:848 %i = phi i32 [ 0, %entry ], [ %inc, %for.cond ]849 %iprom = sext i32 %i to i64850 %b = getelementptr inbounds i16, ptr %p, i64 %iprom851 store i16 0, ptr %b, align 4852 %inc = add nsw i32 %i, 1853 switch i32 %i, label %for.cond [854 i32 2096, label %if.end855 i32 2097, label %if.end856 ]857 858if.end:859 ret i32 %i860}861 862; multiple exit case but with a switch as multiple exiting edges from863; a single block is a commonly missed edge case864define i32 @multiple_exit_switch2(ptr %p, i32 %n) {865; CHECK-LABEL: @multiple_exit_switch2(866; CHECK-NEXT: entry:867; CHECK-NEXT: br label [[FOR_COND:%.*]]868; CHECK: for.cond:869; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND]] ]870; CHECK-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64871; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IPROM]]872; CHECK-NEXT: store i16 0, ptr [[B]], align 4873; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1874; CHECK-NEXT: switch i32 [[I]], label [[FOR_COND]] [875; CHECK-NEXT: i32 2096, label [[IF_END:%.*]]876; CHECK-NEXT: i32 2097, label [[IF_END2:%.*]]877; CHECK-NEXT: ]878; CHECK: if.end:879; CHECK-NEXT: ret i32 0880; CHECK: if.end2:881; CHECK-NEXT: ret i32 1882;883; TAILFOLD-LABEL: @multiple_exit_switch2(884; TAILFOLD-NEXT: entry:885; TAILFOLD-NEXT: br label [[FOR_COND:%.*]]886; TAILFOLD: for.cond:887; TAILFOLD-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND]] ]888; TAILFOLD-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64889; TAILFOLD-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IPROM]]890; TAILFOLD-NEXT: store i16 0, ptr [[B]], align 4891; TAILFOLD-NEXT: [[INC]] = add nsw i32 [[I]], 1892; TAILFOLD-NEXT: switch i32 [[I]], label [[FOR_COND]] [893; TAILFOLD-NEXT: i32 2096, label [[IF_END:%.*]]894; TAILFOLD-NEXT: i32 2097, label [[IF_END2:%.*]]895; TAILFOLD-NEXT: ]896; TAILFOLD: if.end:897; TAILFOLD-NEXT: ret i32 0898; TAILFOLD: if.end2:899; TAILFOLD-NEXT: ret i32 1900;901entry:902 br label %for.cond903 904for.cond:905 %i = phi i32 [ 0, %entry ], [ %inc, %for.cond ]906 %iprom = sext i32 %i to i64907 %b = getelementptr inbounds i16, ptr %p, i64 %iprom908 store i16 0, ptr %b, align 4909 %inc = add nsw i32 %i, 1910 switch i32 %i, label %for.cond [911 i32 2096, label %if.end912 i32 2097, label %if.end2913 ]914 915if.end:916 ret i32 0917 918if.end2:919 ret i32 1920}921 922define i32 @multiple_latch1(ptr %p) {923; CHECK-LABEL: @multiple_latch1(924; CHECK-NEXT: entry:925; CHECK-NEXT: br label [[FOR_BODY:%.*]]926; CHECK: for.body:927; CHECK-NEXT: [[I_02:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY_BACKEDGE:%.*]] ]928; CHECK-NEXT: [[INC]] = add nsw i32 [[I_02]], 1929; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[INC]], 16930; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY_BACKEDGE]], label [[FOR_SECOND:%.*]]931; CHECK: for.second:932; CHECK-NEXT: [[IPROM:%.*]] = sext i32 [[I_02]] to i64933; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IPROM]]934; CHECK-NEXT: store i16 0, ptr [[B]], align 4935; CHECK-NEXT: [[CMPS:%.*]] = icmp sgt i32 [[INC]], 16936; CHECK-NEXT: br i1 [[CMPS]], label [[FOR_BODY_BACKEDGE]], label [[FOR_END:%.*]]937; CHECK: for.body.backedge:938; CHECK-NEXT: br label [[FOR_BODY]]939; CHECK: for.end:940; CHECK-NEXT: ret i32 0941;942; TAILFOLD-LABEL: @multiple_latch1(943; TAILFOLD-NEXT: entry:944; TAILFOLD-NEXT: br label [[FOR_BODY:%.*]]945; TAILFOLD: for.body:946; TAILFOLD-NEXT: [[I_02:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY_BACKEDGE:%.*]] ]947; TAILFOLD-NEXT: [[INC]] = add nsw i32 [[I_02]], 1948; TAILFOLD-NEXT: [[CMP:%.*]] = icmp slt i32 [[INC]], 16949; TAILFOLD-NEXT: br i1 [[CMP]], label [[FOR_BODY_BACKEDGE]], label [[FOR_SECOND:%.*]]950; TAILFOLD: for.second:951; TAILFOLD-NEXT: [[IPROM:%.*]] = sext i32 [[I_02]] to i64952; TAILFOLD-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IPROM]]953; TAILFOLD-NEXT: store i16 0, ptr [[B]], align 4954; TAILFOLD-NEXT: [[CMPS:%.*]] = icmp sgt i32 [[INC]], 16955; TAILFOLD-NEXT: br i1 [[CMPS]], label [[FOR_BODY_BACKEDGE]], label [[FOR_END:%.*]]956; TAILFOLD: for.body.backedge:957; TAILFOLD-NEXT: br label [[FOR_BODY]]958; TAILFOLD: for.end:959; TAILFOLD-NEXT: ret i32 0960;961entry:962 br label %for.body963 964for.body:965 %i.02 = phi i32 [ 0, %entry ], [ %inc, %for.body.backedge]966 %inc = add nsw i32 %i.02, 1967 %cmp = icmp slt i32 %inc, 16968 br i1 %cmp, label %for.body.backedge, label %for.second969 970for.second:971 %iprom = sext i32 %i.02 to i64972 %b = getelementptr inbounds i16, ptr %p, i64 %iprom973 store i16 0, ptr %b, align 4974 %cmps = icmp sgt i32 %inc, 16975 br i1 %cmps, label %for.body.backedge, label %for.end976 977for.body.backedge:978 br label %for.body979 980for.end:981 ret i32 0982}983 984 985; two back branches - loop simplify with convert this to the same form986; as previous before vectorizer sees it, but show that.987define i32 @multiple_latch2(ptr %p) {988; CHECK-LABEL: @multiple_latch2(989; CHECK-NEXT: entry:990; CHECK-NEXT: br label [[FOR_BODY:%.*]]991; CHECK: for.body:992; CHECK-NEXT: [[I_02:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY_BACKEDGE:%.*]] ]993; CHECK-NEXT: [[INC]] = add nsw i32 [[I_02]], 1994; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[INC]], 16995; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY_BACKEDGE]], label [[FOR_SECOND:%.*]]996; CHECK: for.body.backedge:997; CHECK-NEXT: br label [[FOR_BODY]]998; CHECK: for.second:999; CHECK-NEXT: [[IPROM:%.*]] = sext i32 [[I_02]] to i641000; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IPROM]]1001; CHECK-NEXT: store i16 0, ptr [[B]], align 41002; CHECK-NEXT: [[CMPS:%.*]] = icmp sgt i32 [[INC]], 161003; CHECK-NEXT: br i1 [[CMPS]], label [[FOR_BODY_BACKEDGE]], label [[FOR_END:%.*]]1004; CHECK: for.end:1005; CHECK-NEXT: ret i32 01006;1007; TAILFOLD-LABEL: @multiple_latch2(1008; TAILFOLD-NEXT: entry:1009; TAILFOLD-NEXT: br label [[FOR_BODY:%.*]]1010; TAILFOLD: for.body:1011; TAILFOLD-NEXT: [[I_02:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY_BACKEDGE:%.*]] ]1012; TAILFOLD-NEXT: [[INC]] = add nsw i32 [[I_02]], 11013; TAILFOLD-NEXT: [[CMP:%.*]] = icmp slt i32 [[INC]], 161014; TAILFOLD-NEXT: br i1 [[CMP]], label [[FOR_BODY_BACKEDGE]], label [[FOR_SECOND:%.*]]1015; TAILFOLD: for.body.backedge:1016; TAILFOLD-NEXT: br label [[FOR_BODY]]1017; TAILFOLD: for.second:1018; TAILFOLD-NEXT: [[IPROM:%.*]] = sext i32 [[I_02]] to i641019; TAILFOLD-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[IPROM]]1020; TAILFOLD-NEXT: store i16 0, ptr [[B]], align 41021; TAILFOLD-NEXT: [[CMPS:%.*]] = icmp sgt i32 [[INC]], 161022; TAILFOLD-NEXT: br i1 [[CMPS]], label [[FOR_BODY_BACKEDGE]], label [[FOR_END:%.*]]1023; TAILFOLD: for.end:1024; TAILFOLD-NEXT: ret i32 01025;1026entry:1027 br label %for.body1028 1029for.body:1030 %i.02 = phi i32 [ 0, %entry ], [ %inc, %for.body ], [%inc, %for.second]1031 %inc = add nsw i32 %i.02, 11032 %cmp = icmp slt i32 %inc, 161033 br i1 %cmp, label %for.body, label %for.second1034 1035for.second:1036 %iprom = sext i32 %i.02 to i641037 %b = getelementptr inbounds i16, ptr %p, i64 %iprom1038 store i16 0, ptr %b, align 41039 %cmps = icmp sgt i32 %inc, 161040 br i1 %cmps, label %for.body, label %for.end1041 1042for.end:1043 ret i32 01044}1045 1046 1047; Check interaction between block predication and early exits. We need the1048; condition on the early exit to remain dead (i.e. not be used when forming1049; the predicate mask).1050define void @scalar_predication(ptr %addr) {1051; CHECK-LABEL: @scalar_predication(1052; CHECK-NEXT: entry:1053; CHECK-NEXT: br label [[VECTOR_PH:%.*]]1054; CHECK: vector.ph:1055; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1056; CHECK: vector.body:1057; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]1058; CHECK-NEXT: [[TMP1:%.*]] = getelementptr float, ptr [[ADDR:%.*]], i64 [[INDEX]]1059; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP1]], align 41060; CHECK-NEXT: [[TMP4:%.*]] = fcmp une <2 x float> [[WIDE_LOAD]], zeroinitializer1061; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 01062; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]1063; CHECK: pred.store.if:1064; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 01065; CHECK-NEXT: [[TMP6:%.*]] = getelementptr float, ptr [[ADDR]], i64 [[TMP0]]1066; CHECK-NEXT: store float 1.000000e+01, ptr [[TMP6]], align 41067; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]1068; CHECK: pred.store.continue:1069; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP4]], i32 11070; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]1071; CHECK: pred.store.if1:1072; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 11073; CHECK-NEXT: [[TMP9:%.*]] = getelementptr float, ptr [[ADDR]], i64 [[TMP8]]1074; CHECK-NEXT: store float 1.000000e+01, ptr [[TMP9]], align 41075; CHECK-NEXT: br label [[PRED_STORE_CONTINUE2]]1076; CHECK: pred.store.continue2:1077; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 21078; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 2001079; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]1080; CHECK: middle.block:1081; CHECK-NEXT: br label [[SCALAR_PH:%.*]]1082; CHECK: scalar.ph:1083; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]1084; CHECK: loop.header:1085; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 200, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]1086; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[ADDR]], i64 [[IV]]1087; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], 2001088; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[LOOP_BODY:%.*]]1089; CHECK: loop.body:1090; CHECK-NEXT: [[TMP11:%.*]] = load float, ptr [[GEP]], align 41091; CHECK-NEXT: [[PRED:%.*]] = fcmp oeq float [[TMP11]], 0.000000e+001092; CHECK-NEXT: br i1 [[PRED]], label [[LOOP_LATCH]], label [[THEN:%.*]]1093; CHECK: then:1094; CHECK-NEXT: store float 1.000000e+01, ptr [[GEP]], align 41095; CHECK-NEXT: br label [[LOOP_LATCH]]1096; CHECK: loop.latch:1097; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 11098; CHECK-NEXT: br label [[LOOP_HEADER]], !llvm.loop [[LOOP21:![0-9]+]]1099; CHECK: exit:1100; CHECK-NEXT: ret void1101;1102; TAILFOLD-LABEL: @scalar_predication(1103; TAILFOLD-NEXT: entry:1104; TAILFOLD-NEXT: br label [[LOOP_HEADER:%.*]]1105; TAILFOLD: loop.header:1106; TAILFOLD-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]1107; TAILFOLD-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[ADDR:%.*]], i64 [[IV]]1108; TAILFOLD-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], 2001109; TAILFOLD-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[LOOP_BODY:%.*]]1110; TAILFOLD: loop.body:1111; TAILFOLD-NEXT: [[TMP0:%.*]] = load float, ptr [[GEP]], align 41112; TAILFOLD-NEXT: [[PRED:%.*]] = fcmp oeq float [[TMP0]], 0.000000e+001113; TAILFOLD-NEXT: br i1 [[PRED]], label [[LOOP_LATCH]], label [[THEN:%.*]]1114; TAILFOLD: then:1115; TAILFOLD-NEXT: store float 1.000000e+01, ptr [[GEP]], align 41116; TAILFOLD-NEXT: br label [[LOOP_LATCH]]1117; TAILFOLD: loop.latch:1118; TAILFOLD-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 11119; TAILFOLD-NEXT: br label [[LOOP_HEADER]]1120; TAILFOLD: exit:1121; TAILFOLD-NEXT: ret void1122;1123entry:1124 br label %loop.header1125 1126loop.header:1127 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]1128 %gep = getelementptr float, ptr %addr, i64 %iv1129 %exitcond.not = icmp eq i64 %iv, 2001130 br i1 %exitcond.not, label %exit, label %loop.body1131 1132loop.body:1133 %0 = load float, ptr %gep, align 41134 %pred = fcmp oeq float %0, 0.01135 br i1 %pred, label %loop.latch, label %then1136 1137then:1138 store float 10.0, ptr %gep, align 41139 br label %loop.latch1140 1141loop.latch:1142 %iv.next = add nuw nsw i64 %iv, 11143 br label %loop.header1144 1145exit:1146 ret void1147}1148 1149define i32 @me_reduction(ptr %addr) {1150; CHECK-LABEL: @me_reduction(1151; CHECK-NEXT: entry:1152; CHECK-NEXT: br label [[VECTOR_PH:%.*]]1153; CHECK: vector.ph:1154; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1155; CHECK: vector.body:1156; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1157; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]1158; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[ADDR:%.*]], i64 [[INDEX]]1159; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 41160; CHECK-NEXT: [[TMP3]] = add <2 x i32> [[VEC_PHI]], [[WIDE_LOAD]]1161; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 21162; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 2001163; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]1164; CHECK: middle.block:1165; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP3]])1166; CHECK-NEXT: br label [[SCALAR_PH:%.*]]1167; CHECK: scalar.ph:1168; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]1169; CHECK: loop.header:1170; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 200, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]1171; CHECK-NEXT: [[ACCUM:%.*]] = phi i32 [ [[TMP5]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LOOP_LATCH]] ]1172; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[ADDR]], i64 [[IV]]1173; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], 2001174; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[LOOP_LATCH]]1175; CHECK: loop.latch:1176; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[GEP]], align 41177; CHECK-NEXT: [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[TMP6]]1178; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 11179; CHECK-NEXT: [[EXITCOND2_NOT:%.*]] = icmp eq i64 [[IV]], 4001180; CHECK-NEXT: br i1 [[EXITCOND2_NOT]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP23:![0-9]+]]1181; CHECK: exit:1182; CHECK-NEXT: [[LCSSA:%.*]] = phi i32 [ 0, [[LOOP_HEADER]] ], [ [[ACCUM_NEXT]], [[LOOP_LATCH]] ]1183; CHECK-NEXT: ret i32 [[LCSSA]]1184;1185; TAILFOLD-LABEL: @me_reduction(1186; TAILFOLD-NEXT: entry:1187; TAILFOLD-NEXT: br label [[LOOP_HEADER:%.*]]1188; TAILFOLD: loop.header:1189; TAILFOLD-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]1190; TAILFOLD-NEXT: [[ACCUM:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ACCUM_NEXT:%.*]], [[LOOP_LATCH]] ]1191; TAILFOLD-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[ADDR:%.*]], i64 [[IV]]1192; TAILFOLD-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], 2001193; TAILFOLD-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[LOOP_LATCH]]1194; TAILFOLD: loop.latch:1195; TAILFOLD-NEXT: [[TMP0:%.*]] = load i32, ptr [[GEP]], align 41196; TAILFOLD-NEXT: [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[TMP0]]1197; TAILFOLD-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 11198; TAILFOLD-NEXT: [[EXITCOND2_NOT:%.*]] = icmp eq i64 [[IV]], 4001199; TAILFOLD-NEXT: br i1 [[EXITCOND2_NOT]], label [[EXIT]], label [[LOOP_HEADER]]1200; TAILFOLD: exit:1201; TAILFOLD-NEXT: [[LCSSA:%.*]] = phi i32 [ 0, [[LOOP_HEADER]] ], [ [[ACCUM_NEXT]], [[LOOP_LATCH]] ]1202; TAILFOLD-NEXT: ret i32 [[LCSSA]]1203;1204entry:1205 br label %loop.header1206 1207loop.header:1208 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]1209 %accum = phi i32 [0, %entry], [%accum.next, %loop.latch]1210 %gep = getelementptr i32, ptr %addr, i64 %iv1211 %exitcond.not = icmp eq i64 %iv, 2001212 br i1 %exitcond.not, label %exit, label %loop.latch1213 1214loop.latch:1215 %0 = load i32, ptr %gep, align 41216 %accum.next = add i32 %accum, %01217 %iv.next = add nuw nsw i64 %iv, 11218 %exitcond2.not = icmp eq i64 %iv, 4001219 br i1 %exitcond2.not, label %exit, label %loop.header1220 1221exit:1222 %lcssa = phi i32 [0, %loop.header], [%accum.next, %loop.latch]1223 ret i32 %lcssa1224}1225 1226; TODO: The current definition of reduction is too strict, we can vectorize1227; this. There's an analogous single exit case where we extract the N-11228; value of the reduction that we can also handle. If we fix the later, the1229; multiple exit case probably falls out.1230define i32 @me_reduction2(ptr %addr) {1231; CHECK-LABEL: @me_reduction2(1232; CHECK-NEXT: entry:1233; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]1234; CHECK: loop.header:1235; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]1236; CHECK-NEXT: [[ACCUM:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ACCUM_NEXT:%.*]], [[LOOP_LATCH]] ]1237; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[ADDR:%.*]], i64 [[IV]]1238; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], 2001239; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[LOOP_LATCH]]1240; CHECK: loop.latch:1241; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[GEP]], align 41242; CHECK-NEXT: [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[TMP0]]1243; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 11244; CHECK-NEXT: br label [[LOOP_HEADER]]1245; CHECK: exit:1246; CHECK-NEXT: [[ACCUM_LCSSA:%.*]] = phi i32 [ [[ACCUM]], [[LOOP_HEADER]] ]1247; CHECK-NEXT: ret i32 [[ACCUM_LCSSA]]1248;1249; TAILFOLD-LABEL: @me_reduction2(1250; TAILFOLD-NEXT: entry:1251; TAILFOLD-NEXT: br label [[LOOP_HEADER:%.*]]1252; TAILFOLD: loop.header:1253; TAILFOLD-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]1254; TAILFOLD-NEXT: [[ACCUM:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ACCUM_NEXT:%.*]], [[LOOP_LATCH]] ]1255; TAILFOLD-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[ADDR:%.*]], i64 [[IV]]1256; TAILFOLD-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], 2001257; TAILFOLD-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[LOOP_LATCH]]1258; TAILFOLD: loop.latch:1259; TAILFOLD-NEXT: [[TMP0:%.*]] = load i32, ptr [[GEP]], align 41260; TAILFOLD-NEXT: [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[TMP0]]1261; TAILFOLD-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 11262; TAILFOLD-NEXT: br label [[LOOP_HEADER]]1263; TAILFOLD: exit:1264; TAILFOLD-NEXT: [[ACCUM_LCSSA:%.*]] = phi i32 [ [[ACCUM]], [[LOOP_HEADER]] ]1265; TAILFOLD-NEXT: ret i32 [[ACCUM_LCSSA]]1266;1267entry:1268 br label %loop.header1269 1270loop.header:1271 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]1272 %accum = phi i32 [0, %entry], [%accum.next, %loop.latch]1273 %gep = getelementptr i32, ptr %addr, i64 %iv1274 %exitcond.not = icmp eq i64 %iv, 2001275 br i1 %exitcond.not, label %exit, label %loop.latch1276 1277loop.latch:1278 %0 = load i32, ptr %gep, align 41279 %accum.next = add i32 %accum, %01280 %iv.next = add nuw nsw i64 %iv, 11281 br label %loop.header1282 1283exit:1284 ret i32 %accum1285}1286 1287define i16 @multiple_exit_none_via_latch(ptr %dst, i64 %x) {1288; CHECK-LABEL: @multiple_exit_none_via_latch(1289; CHECK-NEXT: entry:1290; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[X:%.*]], i64 100)1291; CHECK-NEXT: [[TMP0:%.*]] = add nuw nsw i64 [[UMIN]], 11292; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP0]], 21293; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1294; CHECK: vector.ph:1295; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 21296; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[N_MOD_VF]], 01297; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i64 2, i64 [[N_MOD_VF]]1298; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP2]]1299; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1300; CHECK: vector.body:1301; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1302; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 01303; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 11304; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[TMP3]]1305; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP4]]1306; CHECK-NEXT: store i64 0, ptr [[TMP5]], align 81307; CHECK-NEXT: store i64 0, ptr [[TMP6]], align 81308; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 21309; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1310; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]1311; CHECK: middle.block:1312; CHECK-NEXT: br label [[SCALAR_PH]]1313; CHECK: scalar.ph:1314; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]1315; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]1316; CHECK: loop.header:1317; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]1318; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]1319; CHECK-NEXT: store i64 0, ptr [[GEP]], align 81320; CHECK-NEXT: [[CMP120:%.*]] = icmp slt i64 [[IV]], 1001321; CHECK-NEXT: br i1 [[CMP120]], label [[LOOP_THEN:%.*]], label [[EXIT_2:%.*]]1322; CHECK: loop.then:1323; CHECK-NEXT: [[CMP3:%.*]] = icmp ne i64 [[IV]], [[X]]1324; CHECK-NEXT: br i1 [[CMP3]], label [[LOOP_LATCH]], label [[EXIT_1:%.*]]1325; CHECK: loop.latch:1326; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 11327; CHECK-NEXT: br label [[LOOP_HEADER]], !llvm.loop [[LOOP25:![0-9]+]]1328; CHECK: exit.1:1329; CHECK-NEXT: ret i16 01330; CHECK: exit.2:1331; CHECK-NEXT: ret i16 11332;1333; TAILFOLD-LABEL: @multiple_exit_none_via_latch(1334; TAILFOLD-NEXT: entry:1335; TAILFOLD-NEXT: br label [[LOOP_HEADER:%.*]]1336; TAILFOLD: loop.header:1337; TAILFOLD-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]1338; TAILFOLD-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[IV]]1339; TAILFOLD-NEXT: store i64 0, ptr [[GEP]], align 81340; TAILFOLD-NEXT: [[CMP120:%.*]] = icmp slt i64 [[IV]], 1001341; TAILFOLD-NEXT: br i1 [[CMP120]], label [[LOOP_THEN:%.*]], label [[EXIT_2:%.*]]1342; TAILFOLD: loop.then:1343; TAILFOLD-NEXT: [[CMP3:%.*]] = icmp ne i64 [[IV]], [[X:%.*]]1344; TAILFOLD-NEXT: br i1 [[CMP3]], label [[LOOP_LATCH]], label [[EXIT_1:%.*]]1345; TAILFOLD: loop.latch:1346; TAILFOLD-NEXT: [[IV_NEXT]] = add i64 [[IV]], 11347; TAILFOLD-NEXT: br label [[LOOP_HEADER]]1348; TAILFOLD: exit.1:1349; TAILFOLD-NEXT: ret i16 01350; TAILFOLD: exit.2:1351; TAILFOLD-NEXT: ret i16 11352;1353entry:1354 br label %loop.header1355 1356loop.header:1357 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]1358 %gep = getelementptr inbounds i32, ptr %dst, i64 %iv1359 store i64 0, ptr %gep1360 %cmp120 = icmp slt i64 %iv, 1001361 br i1 %cmp120, label %loop.then, label %exit.21362 1363loop.then:1364 %cmp3 = icmp ne i64 %iv, %x1365 br i1 %cmp3, label %loop.latch, label %exit.11366 1367loop.latch:1368 %iv.next = add i64 %iv, 11369 br label %loop.header1370 1371exit.1:1372 ret i16 01373 1374exit.2:1375 ret i16 11376}1377 1378; Loop with a switch terminator in the latch block. Cannot be vectorized1379; currently.1380; Test case for https://github.com/llvm/llvm-project/issues/156894.1381define void @switch_in_latch(ptr %a) {1382; CHECK-LABEL: @switch_in_latch(1383; CHECK-NEXT: entry:1384; CHECK-NEXT: br label [[LOOP:%.*]]1385; CHECK: loop:1386; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]1387; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[A:%.*]], i32 [[IV]]1388; CHECK-NEXT: store i32 1, ptr [[GEP]], align 41389; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 11390; CHECK-NEXT: switch i32 [[IV_NEXT]], label [[LOOP]] [1391; CHECK-NEXT: i32 100, label [[EXIT:%.*]]1392; CHECK-NEXT: ]1393; CHECK: exit:1394; CHECK-NEXT: ret void1395;1396; TAILFOLD-LABEL: @switch_in_latch(1397; TAILFOLD-NEXT: entry:1398; TAILFOLD-NEXT: br label [[LOOP:%.*]]1399; TAILFOLD: loop:1400; TAILFOLD-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]1401; TAILFOLD-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[A:%.*]], i32 [[IV]]1402; TAILFOLD-NEXT: store i32 1, ptr [[GEP]], align 41403; TAILFOLD-NEXT: [[IV_NEXT]] = add i32 [[IV]], 11404; TAILFOLD-NEXT: switch i32 [[IV_NEXT]], label [[LOOP]] [1405; TAILFOLD-NEXT: i32 100, label [[EXIT:%.*]]1406; TAILFOLD-NEXT: ]1407; TAILFOLD: exit:1408; TAILFOLD-NEXT: ret void1409;1410entry:1411 br label %loop1412 1413loop:1414 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]1415 %gep = getelementptr i32, ptr %a, i32 %iv1416 store i32 1, ptr %gep, align 41417 %iv.next = add i32 %iv, 11418 switch i32 %iv.next, label %loop [i32 100, label %exit]1419 1420exit:1421 ret void1422}1423