1228 lines · plain
1; REQUIRES: asserts2 3; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -force-widen-divrem-via-safe-divisor=0 -debug -disable-output %s 2>&1 | FileCheck %s4 5target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"6 7@a = common global [2048 x i32] zeroinitializer, align 168@b = common global [2048 x i32] zeroinitializer, align 169@c = common global [2048 x i32] zeroinitializer, align 1610 11 12; CHECK-LABEL: LV: Checking a loop in 'sink1'13; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {14; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF15; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF16; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count17; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count18; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count19; CHECK-EMPTY:20; CHECK-NEXT: ir-bb<entry>:21; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>22; CHECK-NEXT: Successor(s): scalar.ph, vector.ph23; CHECK-EMPTY:24; CHECK-NEXT: vector.ph:25; CHECK-NEXT: Successor(s): vector loop26; CHECK-EMPTY:27; CHECK-NEXT: <x1> vector loop: {28; CHECK-NEXT: vector.body:29; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION30; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VF]]>31; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>32; CHECK-NEXT: WIDEN ir<%cond> = icmp eq ir<%iv>, ir<%x>33; CHECK-NEXT: EMIT vp<[[AND:%.+]]> = logical-and vp<[[MASK]]>, ir<%cond>34; CHECK-NEXT: Successor(s): pred.store35 36; CHECK: <xVFxUF> pred.store: {37; CHECK-NEXT: pred.store.entry:38; CHECK-NEXT: BRANCH-ON-MASK vp<[[AND]]>39; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue40 41; CHECK: pred.store.if:42; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>43; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>44; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>45; CHECK-NEXT: REPLICATE ir<%add> = add ir<%lv.b>, ir<10>46; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]47; CHECK-NEXT: REPLICATE ir<%mul> = mul ir<2>, ir<%add>48; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.a>49; CHECK-NEXT: Successor(s): pred.store.continue50 51; CHECK: pred.store.continue:52; CHECK-NEXT: No successors53; CHECK-NEXT: }54 55; CHECK: if.1:56; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>57; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>58; CHECK-NEXT: No successors59; CHECK-NEXT: }60;61define void @sink1(i32 %k, i32 %x) {62entry:63 br label %loop64 65loop:66 %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]67 %cond = icmp eq i32 %iv, %x68 br i1 %cond, label %if, label %latch69 70if:71 %gep.b = getelementptr inbounds [2048 x i32], ptr @b, i32 0, i32 %iv72 %lv.b = load i32, ptr %gep.b, align 473 %add = add i32 %lv.b, 1074 %mul = mul i32 2, %add75 %gep.a = getelementptr inbounds [2048 x i32], ptr @a, i32 0, i32 %iv76 store i32 %mul, ptr %gep.a, align 477 br label %latch78 79latch:80 %iv.next = add i32 %iv, 181 %large = icmp sge i32 %iv, 882 %exitcond = icmp eq i32 %iv, %k83 %realexit = or i1 %large, %exitcond84 br i1 %realexit, label %exit, label %loop85 86exit:87 ret void88}89 90; CHECK-LABEL: LV: Checking a loop in 'sink2'91; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {92; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF93; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF94; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count95; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count96; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count97; CHECK-EMPTY:98; CHECK-NEXT: ir-bb<entry>:99; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>100; CHECK-NEXT: Successor(s): scalar.ph, vector.ph101; CHECK-EMPTY:102; CHECK-NEXT: vector.ph:103; CHECK-NEXT: Successor(s): vector loop104; CHECK-EMPTY:105; CHECK-NEXT: <x1> vector loop: {106; CHECK-NEXT: vector.body:107; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION108; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VF]]>109; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>110; CHECK-NEXT: Successor(s): pred.load111 112; CHECK: <xVFxUF> pred.load: {113; CHECK-NEXT: pred.load.entry:114; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>115; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue116 117; CHECK: pred.load.if:118; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>119; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>120; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>121; CHECK-NEXT: Successor(s): pred.load.continue122 123; CHECK: pred.load.continue:124; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>125; CHECK-NEXT: No successors126; CHECK-NEXT: }127 128; CHECK: loop.0:129; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<2>130; CHECK-NEXT: Successor(s): pred.store131 132; CHECK: <xVFxUF> pred.store: {133; CHECK-NEXT: pred.store.entry:134; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>135; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue136 137; CHECK: pred.store.if:138; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>139; CHECK-NEXT: REPLICATE ir<%add> = add vp<[[PRED]]>, ir<10>140; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.a>141; CHECK-NEXT: Successor(s): pred.store.continue142 143; CHECK: pred.store.continue:144; CHECK-NEXT: No successors145; CHECK-NEXT: }146 147; CHECK: loop.1:148; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>149; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>150; CHECK-NEXT: No successors151; CHECK-NEXT: }152;153define void @sink2(i32 %k) {154entry:155 br label %loop156 157loop:158 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]159 %gep.b = getelementptr inbounds [2048 x i32], ptr @b, i32 0, i32 %iv160 %lv.b = load i32, ptr %gep.b, align 4161 %add = add i32 %lv.b, 10162 %mul = mul i32 %iv, 2163 %gep.a = getelementptr inbounds [2048 x i32], ptr @a, i32 0, i32 %mul164 store i32 %add, ptr %gep.a, align 4165 %iv.next = add i32 %iv, 1166 %large = icmp sge i32 %iv, 8167 %exitcond = icmp eq i32 %iv, %k168 %realexit = or i1 %large, %exitcond169 br i1 %realexit, label %exit, label %loop170 171exit:172 ret void173}174 175; CHECK-LABEL: LV: Checking a loop in 'sink3'176; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {177; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF178; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF179; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count180; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count181; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count182; CHECK-EMPTY:183; CHECK-NEXT: ir-bb<entry>:184; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>185; CHECK-NEXT: Successor(s): scalar.ph, vector.ph186; CHECK-EMPTY:187; CHECK-NEXT: vector.ph:188; CHECK-NEXT: Successor(s): vector loop189; CHECK-EMPTY:190; CHECK-NEXT: <x1> vector loop: {191; CHECK-NEXT: vector.body:192; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION193; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VF]]>194; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>195; CHECK-NEXT: Successor(s): pred.load196 197; CHECK: <xVFxUF> pred.load: {198; CHECK-NEXT: pred.load.entry:199; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>200; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue201 202; CHECK: pred.load.if:203; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>204; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>205; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V)206; CHECK-NEXT: Successor(s): pred.load.continue207 208; CHECK: pred.load.continue:209; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>210; CHECK-NEXT: No successors211; CHECK-NEXT: }212 213; CHECK: loop.0:214; CHECK-NEXT: WIDEN ir<%add> = add vp<[[PRED]]>, ir<10>215; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<%add>216; CHECK-NEXT: Successor(s): pred.store217 218; CHECK: <xVFxUF> pred.store: {219; CHECK-NEXT: pred.store.entry:220; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>221; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue222 223; CHECK: pred.store.if:224; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>225; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.a>226; CHECK-NEXT: Successor(s): pred.store.continue227 228; CHECK: pred.store.continue:229; CHECK-NEXT: No successors230; CHECK-NEXT: }231 232; CHECK: loop.1:233; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>234; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>235; CHECK-NEXT: No successors236; CHECK-NEXT: }237;238define void @sink3(i32 %k) {239entry:240 br label %loop241 242loop:243 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]244 %gep.b = getelementptr inbounds [2048 x i32], ptr @b, i32 0, i32 %iv245 %lv.b = load i32, ptr %gep.b, align 4246 %add = add i32 %lv.b, 10247 %mul = mul i32 %iv, %add248 %gep.a = getelementptr inbounds [2048 x i32], ptr @a, i32 0, i32 %mul249 store i32 %add, ptr %gep.a, align 4250 %iv.next = add i32 %iv, 1251 %large = icmp sge i32 %iv, 8252 %exitcond = icmp eq i32 %iv, %k253 %realexit = or i1 %large, %exitcond254 br i1 %realexit, label %exit, label %loop255 256exit:257 ret void258}259 260; Make sure we do not sink uniform instructions.261define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) {262; CHECK-LABEL: LV: Checking a loop in 'uniform_gep'263; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {264; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF265; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF266; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count267; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count268; CHECK-NEXT: Live-in ir<11> = original trip-count269; CHECK-EMPTY:270; CHECK-NEXT: ir-bb<entry>:271; CHECK-NEXT: Successor(s): scalar.ph, vector.ph272; CHECK-EMPTY:273; CHECK-NEXT: vector.ph:274; CHECK-NEXT: Successor(s): vector loop275; CHECK-EMPTY:276; CHECK-NEXT: <x1> vector loop: {277; CHECK-NEXT: vector.body:278; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION279; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<21>, ir<1>, vp<[[VF]]>280; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<21> + vp<[[CAN_IV]]> * ir<1>281; CHECK-NEXT: EMIT vp<[[WIDE_CAN_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]>282; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDE_CAN_IV]]>, vp<[[BTC]]>283; CHECK-NEXT: CLONE ir<%lv> = load ir<%A>284; CHECK-NEXT: WIDEN ir<%cmp> = icmp uge ir<%iv>, ir<%k>285; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = logical-and vp<[[MASK]]>, ir<%cmp>286; CHECK-NEXT: Successor(s): pred.store287; CHECK-EMPTY:288; CHECK-NEXT: <xVFxUF> pred.store: {289; CHECK-NEXT: pred.store.entry:290; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]>291; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue292; CHECK-EMPTY:293; CHECK-NEXT: pred.store.if:294; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1>295; CHECK-NEXT: REPLICATE ir<%gep.B> = getelementptr inbounds ir<%B>, vp<[[STEPS]]>296; CHECK-NEXT: REPLICATE store ir<%lv>, ir<%gep.B>297; CHECK-NEXT: Successor(s): pred.store.continue298; CHECK-EMPTY:299; CHECK-NEXT: pred.store.continue:300; CHECK-NEXT: No successors301; CHECK-NEXT: }302; CHECK-NEXT: Successor(s): loop.then.0303; CHECK-EMPTY:304; CHECK-NEXT: loop.then.0:305; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>306; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>307; CHECK-NEXT: No successors308; CHECK-NEXT: }309;310entry:311 br label %loop312 313loop:314 %iv = phi i64 [ 21, %entry ], [ %iv.next, %loop.latch ]315 %gep.A.uniform = getelementptr inbounds i16, ptr %A, i64 0316 %gep.B = getelementptr inbounds i16, ptr %B, i64 %iv317 %lv = load i16, ptr %gep.A.uniform, align 1318 %cmp = icmp ult i64 %iv, %k319 br i1 %cmp, label %loop.latch, label %loop.then320 321loop.then:322 store i16 %lv, ptr %gep.B, align 1323 br label %loop.latch324 325loop.latch:326 %iv.next = add nsw i64 %iv, 1327 %cmp179 = icmp slt i64 %iv.next, 32328 br i1 %cmp179, label %loop, label %exit329exit:330 ret void331}332 333; Loop with predicated load.334define void @pred_cfg1(i32 %k, i32 %j) {335; CHECK-LABEL: LV: Checking a loop in 'pred_cfg1'336; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {337; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF338; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF339; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count340; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count341; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count342; CHECK-EMPTY:343; CHECK-NEXT: ir-bb<entry>:344; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>345; CHECK-NEXT: Successor(s): scalar.ph, vector.ph346; CHECK-EMPTY:347; CHECK-NEXT: vector.ph:348; CHECK-NEXT: Successor(s): vector loop349; CHECK-EMPTY:350; CHECK-NEXT: <x1> vector loop: {351; CHECK-NEXT: vector.body:352; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION353; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VF]]>354; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>355; CHECK-NEXT: WIDEN ir<%c.1> = icmp ult ir<%iv>, ir<%j>356; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>357; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = logical-and vp<[[MASK1]]>, ir<%c.1>358; CHECK-NEXT: Successor(s): pred.load359; CHECK-EMPTY:360; CHECK-NEXT: <xVFxUF> pred.load: {361; CHECK-NEXT: pred.load.entry:362; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]>363; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue364; CHECK-EMPTY:365; CHECK-NEXT: pred.load.if:366; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>367; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>368; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V)369; CHECK-NEXT: Successor(s): pred.load.continue370; CHECK-EMPTY:371; CHECK-NEXT: pred.load.continue:372; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>373; CHECK-NEXT: No successors374; CHECK-NEXT: }375; CHECK-NEXT: Successor(s): then.0.0376; CHECK-EMPTY:377; CHECK-NEXT: then.0.0:378; CHECK-NEXT: BLEND ir<%p> = ir<0> vp<[[PRED]]>/ir<%c.1>379; CHECK-NEXT: Successor(s): pred.store380; CHECK-EMPTY:381; CHECK-NEXT: <xVFxUF> pred.store: {382; CHECK-NEXT: pred.store.entry:383; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK1]]>384; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue385; CHECK-EMPTY:386; CHECK-NEXT: pred.store.if:387; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>388; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.a>389; CHECK-NEXT: Successor(s): pred.store.continue390; CHECK-EMPTY:391; CHECK-NEXT: pred.store.continue:392; CHECK-NEXT: No successors393; CHECK-NEXT: }394; CHECK-NEXT: Successor(s): next.0.1395; CHECK-EMPTY:396; CHECK-NEXT: next.0.1:397; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>398; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>399; CHECK-NEXT: No successors400; CHECK-NEXT: }401;402entry:403 br label %loop404 405loop:406 %iv = phi i32 [ 0, %entry ], [ %iv.next, %next.0 ]407 %gep.b = getelementptr inbounds [2048 x i32], ptr @b, i32 0, i32 %iv408 %c.1 = icmp ult i32 %iv, %j409 %mul = mul i32 %iv, 10410 %gep.a = getelementptr inbounds [2048 x i32], ptr @a, i32 0, i32 %mul411 br i1 %c.1, label %then.0, label %next.0412 413then.0:414 %lv.b = load i32, ptr %gep.b, align 4415 br label %next.0416 417next.0:418 %p = phi i32 [ 0, %loop ], [ %lv.b, %then.0 ]419 store i32 %p, ptr %gep.a, align 4420 %iv.next = add i32 %iv, 1421 %large = icmp sge i32 %iv, 8422 %exitcond = icmp eq i32 %iv, %k423 %realexit = or i1 %large, %exitcond424 br i1 %realexit, label %exit, label %loop425 426exit:427 ret void428}429 430; Loop with predicated load and store in separate blocks, store depends on431; loaded value.432define void @pred_cfg2(i32 %k, i32 %j) {433; CHECK-LABEL: LV: Checking a loop in 'pred_cfg2'434; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {435; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF436; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF437; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count438; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count439; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count440; CHECK-EMPTY:441; CHECK-NEXT: ir-bb<entry>:442; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>443; CHECK-NEXT: Successor(s): scalar.ph, vector.ph444; CHECK-EMPTY:445; CHECK-NEXT: vector.ph:446; CHECK-NEXT: Successor(s): vector loop447; CHECK-EMPTY:448; CHECK-NEXT: <x1> vector loop: {449; CHECK-NEXT: vector.body:450; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION451; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VF]]>452; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>453; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>454; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>455; CHECK-NEXT: WIDEN ir<%c.1> = icmp ugt ir<%iv>, ir<%j>456; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = logical-and vp<[[MASK1]]>, ir<%c.0>457; CHECK-NEXT: Successor(s): pred.load458; CHECK-EMPTY:459; CHECK-NEXT: <xVFxUF> pred.load: {460; CHECK-NEXT: pred.load.entry:461; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]>462; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue463; CHECK-EMPTY:464; CHECK-NEXT: pred.load.if:465; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>466; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>467; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V)468; CHECK-NEXT: Successor(s): pred.load.continue469; CHECK-EMPTY:470; CHECK-NEXT: pred.load.continue:471; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>472; CHECK-NEXT: No successors473; CHECK-NEXT: }474; CHECK-NEXT: Successor(s): then.0.0475; CHECK-EMPTY:476; CHECK-NEXT: then.0.0:477; CHECK-NEXT: BLEND ir<%p> = ir<0> vp<[[PRED]]>/ir<%c.0>478; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = logical-and vp<[[MASK1]]>, ir<%c.1>479; CHECK-NEXT: Successor(s): pred.store480; CHECK-EMPTY:481; CHECK-NEXT: <xVFxUF> pred.store: {482; CHECK-NEXT: pred.store.entry:483; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK3]]>484; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue485; CHECK-EMPTY:486; CHECK-NEXT: pred.store.if:487; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>488; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.a>489; CHECK-NEXT: Successor(s): pred.store.continue490; CHECK-EMPTY:491; CHECK-NEXT: pred.store.continue:492; CHECK-NEXT: No successors493; CHECK-NEXT: }494; CHECK-NEXT: Successor(s): then.1.1495; CHECK-EMPTY:496; CHECK-NEXT: then.1.1:497; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>498; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>499; CHECK-NEXT: No successors500; CHECK-NEXT: }501;502entry:503 br label %loop504 505loop:506 %iv = phi i32 [ 0, %entry ], [ %iv.next, %next.1 ]507 %gep.b = getelementptr inbounds [2048 x i32], ptr @b, i32 0, i32 %iv508 %mul = mul i32 %iv, 10509 %gep.a = getelementptr inbounds [2048 x i32], ptr @a, i32 0, i32 %mul510 %c.0 = icmp ult i32 %iv, %j511 %c.1 = icmp ugt i32 %iv, %j512 br i1 %c.0, label %then.0, label %next.0513 514then.0:515 %lv.b = load i32, ptr %gep.b, align 4516 br label %next.0517 518next.0:519 %p = phi i32 [ 0, %loop ], [ %lv.b, %then.0 ]520 br i1 %c.1, label %then.1, label %next.1521 522then.1:523 store i32 %p, ptr %gep.a, align 4524 br label %next.1525 526next.1:527 %iv.next = add i32 %iv, 1528 %large = icmp sge i32 %iv, 8529 %exitcond = icmp eq i32 %iv, %k530 %realexit = or i1 %large, %exitcond531 br i1 %realexit, label %exit, label %loop532 533exit:534 ret void535}536 537; Loop with predicated load and store in separate blocks, store does not depend538; on loaded value.539define void @pred_cfg3(i32 %k, i32 %j) {540; CHECK-LABEL: LV: Checking a loop in 'pred_cfg3'541; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {542; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF543; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF544; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count545; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count546; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count547; CHECK-EMPTY:548; CHECK-NEXT: ir-bb<entry>:549; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>550; CHECK-NEXT: Successor(s): scalar.ph, vector.ph551; CHECK-EMPTY:552; CHECK-NEXT: vector.ph:553; CHECK-NEXT: Successor(s): vector loop554; CHECK-EMPTY:555; CHECK-NEXT: <x1> vector loop: {556; CHECK-NEXT: vector.body:557; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION558; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VF]]>559; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>560; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>561; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>562; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = logical-and vp<[[MASK1:%.+]]>, ir<%c.0>563; CHECK-NEXT: Successor(s): pred.load564; CHECK-EMPTY:565; CHECK-NEXT: <xVFxUF> pred.load: {566; CHECK-NEXT: pred.load.entry:567; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]>568; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue569; CHECK-EMPTY:570; CHECK-NEXT: pred.load.if:571; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>572; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>573; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>574; CHECK-NEXT: Successor(s): pred.load.continue575; CHECK-EMPTY:576; CHECK-NEXT: pred.load.continue:577; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b>578; CHECK-NEXT: No successors579; CHECK-NEXT: }580; CHECK-NEXT: Successor(s): then.0.0581; CHECK-EMPTY:582; CHECK-NEXT: then.0.0:583; CHECK-NEXT: BLEND ir<%p> = ir<0> vp<[[PRED]]>/ir<%c.0>584; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = logical-and vp<[[MASK1]]>, ir<%c.0>585; CHECK-NEXT: Successor(s): pred.store586; CHECK-EMPTY:587; CHECK-NEXT: <xVFxUF> pred.store: {588; CHECK-NEXT: pred.store.entry:589; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK3]]>590; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue591; CHECK-EMPTY:592; CHECK-NEXT: pred.store.if:593; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul>594; CHECK-NEXT: REPLICATE store ir<0>, ir<%gep.a>595; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, ir<%mul>596; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.c>597; CHECK-NEXT: Successor(s): pred.store.continue598; CHECK-EMPTY:599; CHECK-NEXT: pred.store.continue:600; CHECK-NEXT: No successors601; CHECK-NEXT: }602; CHECK-NEXT: Successor(s): then.1.2603; CHECK-EMPTY:604; CHECK-NEXT: then.1.2:605; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>606; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>607; CHECK-NEXT: No successors608; CHECK-NEXT: }609;610entry:611 br label %loop612 613loop:614 %iv = phi i32 [ 0, %entry ], [ %iv.next, %next.1 ]615 %gep.b = getelementptr inbounds [2048 x i32], ptr @b, i32 0, i32 %iv616 %mul = mul i32 %iv, 10617 %gep.a = getelementptr inbounds [2048 x i32], ptr @a, i32 0, i32 %mul618 %gep.c = getelementptr inbounds [2048 x i32], ptr @c, i32 0, i32 %mul619 %c.0 = icmp ult i32 %iv, %j620 br i1 %c.0, label %then.0, label %next.0621 622then.0:623 %lv.b = load i32, ptr %gep.b, align 4624 br label %next.0625 626next.0:627 %p = phi i32 [ 0, %loop ], [ %lv.b, %then.0 ]628 br i1 %c.0, label %then.1, label %next.1629 630then.1:631 store i32 0, ptr %gep.a, align 4632 store i32 %p, ptr %gep.c, align 4633 br label %next.1634 635next.1:636 %iv.next = add i32 %iv, 1637 %large = icmp sge i32 %iv, 8638 %exitcond = icmp eq i32 %iv, %k639 %realexit = or i1 %large, %exitcond640 br i1 %realexit, label %exit, label %loop641 642exit:643 ret void644}645 646define void @merge_3_replicate_region(i32 %k, i32 %j) {647; CHECK-LABEL: LV: Checking a loop in 'merge_3_replicate_region'648; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {649; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF650; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF651; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count652; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count653; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count654; CHECK-EMPTY:655; CHECK-NEXT: ir-bb<entry>:656; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>657; CHECK-NEXT: Successor(s): scalar.ph, vector.ph658; CHECK-EMPTY:659; CHECK-NEXT: vector.ph:660; CHECK-NEXT: Successor(s): vector loop661; CHECK-EMPTY:662; CHECK-NEXT: <x1> vector loop: {663; CHECK-NEXT: vector.body:664; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION665; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VF]]>666; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>667; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>668; CHECK-NEXT: Successor(s): pred.store669; CHECK-EMPTY:670; CHECK-NEXT: <xVFxUF> pred.store: {671; CHECK-NEXT: pred.store.entry:672; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>673; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue674; CHECK-EMPTY:675; CHECK-NEXT: pred.store.if:676; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>677; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a>678; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]>679; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>680; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]>681; CHECK-NEXT: REPLICATE store ir<%lv.a>, ir<%gep.c>682; CHECK-NEXT: REPLICATE store ir<%lv.b>, ir<%gep.a>683; CHECK-NEXT: Successor(s): pred.store.continue684; CHECK-EMPTY:685; CHECK-NEXT: pred.store.continue:686; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv.a>687; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%lv.b>688; CHECK-NEXT: No successors689; CHECK-NEXT: }690; CHECK-NEXT: Successor(s): loop.3691; CHECK-EMPTY:692; CHECK-NEXT: loop.3:693; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>694; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = logical-and vp<[[MASK]]>, ir<%c.0>695; CHECK-NEXT: WIDEN ir<%mul> = mul vp<[[PRED1]]>, vp<[[PRED2]]>696; CHECK-NEXT: Successor(s): pred.store697; CHECK-EMPTY:698; CHECK-NEXT: <xVFxUF> pred.store: {699; CHECK-NEXT: pred.store.entry:700; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]>701; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue702; CHECK-EMPTY:703; CHECK-NEXT: pred.store.if:704; CHECK-NEXT: REPLICATE ir<%gep.c.1> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]>705; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.c.1>706; CHECK-NEXT: Successor(s): pred.store.continue707; CHECK-EMPTY:708; CHECK-NEXT: pred.store.continue:709; CHECK-NEXT: No successors710; CHECK-NEXT: }711; CHECK-NEXT: Successor(s): then.0.4712; CHECK-EMPTY:713; CHECK-NEXT: then.0.4:714; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>715; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>716; CHECK-NEXT: No successors717; CHECK-NEXT: }718;719entry:720 br label %loop721 722loop:723 %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]724 %gep.a = getelementptr inbounds [2048 x i32], ptr @a, i32 0, i32 %iv725 %lv.a = load i32, ptr %gep.a, align 4726 %gep.b = getelementptr inbounds [2048 x i32], ptr @b, i32 0, i32 %iv727 %lv.b = load i32, ptr %gep.b, align 4728 %gep.c = getelementptr inbounds [2048 x i32], ptr @c, i32 0, i32 %iv729 store i32 %lv.a, ptr %gep.c, align 4730 store i32 %lv.b, ptr %gep.a, align 4731 %c.0 = icmp ult i32 %iv, %j732 br i1 %c.0, label %then.0, label %latch733 734then.0:735 %mul = mul i32 %lv.a, %lv.b736 %gep.c.1 = getelementptr inbounds [2048 x i32], ptr @c, i32 0, i32 %iv737 store i32 %mul, ptr %gep.c.1, align 4738 br label %latch739 740latch:741 %iv.next = add i32 %iv, 1742 %large = icmp sge i32 %iv, 8743 %exitcond = icmp eq i32 %iv, %k744 %realexit = or i1 %large, %exitcond745 br i1 %realexit, label %exit, label %loop746 747exit:748 ret void749}750 751 752define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) {753; CHECK-LABEL: LV: Checking a loop in 'update_2_uses_in_same_recipe_in_merged_block'754; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {755; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF756; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF757; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count758; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count759; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count760; CHECK-EMPTY:761; CHECK-NEXT: ir-bb<entry>:762; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>763; CHECK-NEXT: Successor(s): scalar.ph, vector.ph764; CHECK-EMPTY:765; CHECK-NEXT: vector.ph:766; CHECK-NEXT: Successor(s): vector loop767; CHECK-EMPTY:768; CHECK-NEXT: <x1> vector loop: {769; CHECK-NEXT: vector.body:770; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION771; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VF]]>772; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>773; CHECK-NEXT: Successor(s): pred.store774; CHECK-EMPTY:775; CHECK-NEXT: <xVFxUF> pred.store: {776; CHECK-NEXT: pred.store.entry:777; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>778; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue779; CHECK-EMPTY:780; CHECK-NEXT: pred.store.if:781; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>782; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>783; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a>784; CHECK-NEXT: REPLICATE ir<%div> = sdiv ir<%lv.a>, ir<%lv.a>785; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a>786; CHECK-NEXT: Successor(s): pred.store.continue787; CHECK-EMPTY:788; CHECK-NEXT: pred.store.continue:789; CHECK-NEXT: No successors790; CHECK-NEXT: }791; CHECK-NEXT: Successor(s): loop.2792; CHECK-EMPTY:793; CHECK-NEXT: loop.2:794; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>795; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>796; CHECK-NEXT: No successors797; CHECK-NEXT: }798;799entry:800 br label %loop801 802loop:803 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]804 %gep.a = getelementptr inbounds [2048 x i32], ptr @a, i32 0, i32 %iv805 %lv.a = load i32, ptr %gep.a, align 4806 %div = sdiv i32 %lv.a, %lv.a807 store i32 %div, ptr %gep.a, align 4808 %iv.next = add i32 %iv, 1809 %large = icmp sge i32 %iv, 8810 %exitcond = icmp eq i32 %iv, %k811 %realexit = or i1 %large, %exitcond812 br i1 %realexit, label %exit, label %loop813 814exit:815 ret void816}817 818define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {819; CHECK-LABEL: LV: Checking a loop in 'recipe_in_merge_candidate_used_by_first_order_recurrence'820; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {821; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF822; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF823; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count824; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count825; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count826; CHECK-EMPTY:827; CHECK-NEXT: ir-bb<entry>:828; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k))<nuw><nsw>829; CHECK-NEXT: Successor(s): scalar.ph, vector.ph830; CHECK-EMPTY:831; CHECK-NEXT: vector.ph:832; CHECK-NEXT: Successor(s): vector loop833; CHECK-EMPTY:834; CHECK-NEXT: <x1> vector loop: {835; CHECK-NEXT: vector.body:836; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION837; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VF]]>838; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[PRED:%.+]]>839; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>840; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]>841; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]>842; CHECK-NEXT: Successor(s): pred.load843; CHECK-EMPTY:844; CHECK-NEXT: <xVFxUF> pred.load: {845; CHECK-NEXT: pred.load.entry:846; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>847; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue848; CHECK-EMPTY:849; CHECK-NEXT: pred.load.if:850; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a>851; CHECK-NEXT: Successor(s): pred.load.continue852; CHECK-EMPTY:853; CHECK-NEXT: pred.load.continue:854; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED]]> = ir<%lv.a>855; CHECK-NEXT: No successors856; CHECK-NEXT: }857; CHECK-NEXT: Successor(s): loop.0858; CHECK-EMPTY:859; CHECK-NEXT: loop.0:860; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for>, vp<[[PRED]]>861; CHECK-NEXT: Successor(s): pred.store862; CHECK-EMPTY:863; CHECK-NEXT: <xVFxUF> pred.store: {864; CHECK-NEXT: pred.store.entry:865; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>866; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue867; CHECK-EMPTY:868; CHECK-NEXT: pred.store.if:869; CHECK-NEXT: REPLICATE ir<%div> = sdiv vp<[[SPLICE]]>, vp<[[PRED]]>870; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a>871; CHECK-NEXT: Successor(s): pred.store.continue872; CHECK-EMPTY:873; CHECK-NEXT: pred.store.continue:874; CHECK-NEXT: No successors875; CHECK-NEXT: }876; CHECK-NEXT: Successor(s): loop.2877; CHECK-EMPTY:878; CHECK-NEXT: loop.2:879; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>880; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>881; CHECK-NEXT: No successors882; CHECK-NEXT: }883;884entry:885 br label %loop886 887loop:888 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]889 %for = phi i32 [ 0, %entry ], [ %lv.a, %loop ]890 %gep.a = getelementptr inbounds [2048 x i32], ptr @a, i32 0, i32 %iv891 %lv.a = load i32, ptr %gep.a, align 4892 %div = sdiv i32 %for, %lv.a893 store i32 %div, ptr %gep.a, align 4894 %iv.next = add i32 %iv, 1895 %large = icmp sge i32 %iv, 8896 %exitcond = icmp eq i32 %iv, %k897 %realexit = or i1 %large, %exitcond898 br i1 %realexit, label %exit, label %loop899 900exit:901 ret void902}903 904define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) {905; CHECK-LABEL: LV: Checking a loop in 'update_multiple_users'906; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {907; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF908; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count909; CHECK-NEXT: Live-in ir<999> = original trip-count910; CHECK-EMPTY:911; CHECK-NEXT: ir-bb<entry>:912; CHECK-NEXT: Successor(s): scalar.ph, vector.ph913; CHECK-EMPTY:914; CHECK-NEXT: vector.ph:915; CHECK-NEXT: Successor(s): vector loop916; CHECK-EMPTY:917; CHECK-NEXT: <x1> vector loop: {918; CHECK-NEXT: vector.body:919; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION920; CHECK-NEXT: Successor(s): pred.store921; CHECK-EMPTY:922; CHECK-NEXT: <xVFxUF> pred.store: {923; CHECK-NEXT: pred.store.entry:924; CHECK-NEXT: BRANCH-ON-MASK ir<%c>925; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue926; CHECK-EMPTY:927; CHECK-NEXT: pred.store.if:928; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src>929; CHECK-NEXT: REPLICATE ir<%l2> = trunc ir<%l1>930; CHECK-NEXT: REPLICATE ir<%cmp> = icmp eq ir<%l1>, ir<0>931; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2>932; CHECK-NEXT: REPLICATE store ir<%sel>, ir<%dst>933; CHECK-NEXT: Successor(s): pred.store.continue934; CHECK-EMPTY:935; CHECK-NEXT: pred.store.continue:936; CHECK-NEXT: No successors937; CHECK-NEXT: }938; CHECK-NEXT: Successor(s): loop.then.1939; CHECK-EMPTY:940; CHECK-NEXT: loop.then.1:941; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>942; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>943; CHECK-NEXT: No successors944; CHECK-NEXT: }945;946entry:947 br label %loop.header948 949loop.header:950 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]951 br i1 %c, label %loop.then, label %loop.latch952 953loop.then:954 %l1 = load i16, ptr %src, align 2955 %l2 = trunc i16 %l1 to i8956 %cmp = icmp eq i16 %l1, 0957 %sel = select i1 %cmp, i8 5, i8 %l2958 store i8 %sel, ptr %dst, align 1959 %sext.l1 = sext i16 %l1 to i32960 br label %loop.latch961 962loop.latch:963 %iv.next = add nsw i64 %iv, 1964 %ec = icmp eq i64 %iv.next, 999965 br i1 %ec, label %exit, label %loop.header966 967exit:968 ret void969}970 971define void @sinking_requires_duplication(ptr %addr) {972; CHECK-LABEL: LV: Checking a loop in 'sinking_requires_duplication'973; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {974; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF975; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF976; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count977; CHECK-NEXT: Live-in ir<201> = original trip-count978; CHECK-EMPTY:979; CHECK-NEXT: ir-bb<entry>:980; CHECK-NEXT: Successor(s): scalar.ph, vector.ph981; CHECK-EMPTY:982; CHECK-NEXT: vector.ph:983; CHECK-NEXT: Successor(s): vector loop984; CHECK-EMPTY:985; CHECK-NEXT: <x1> vector loop: {986; CHECK-NEXT: vector.body:987; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION988; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>, vp<[[VF]]>989; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]>990; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep>991; CHECK-NEXT: WIDEN ir<%0> = load vp<[[VEC_PTR]]>992; CHECK-NEXT: WIDEN ir<%pred> = fcmp une ir<%0>, ir<0.000000e+00>993; CHECK-NEXT: Successor(s): pred.store994; CHECK-EMPTY:995; CHECK-NEXT: <xVFxUF> pred.store: {996; CHECK-NEXT: pred.store.entry:997; CHECK-NEXT: BRANCH-ON-MASK ir<%pred>998; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue999; CHECK-EMPTY:1000; CHECK-NEXT: pred.store.if:1001; CHECK-NEXT: vp<[[STEPS_SUNK:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>1002; CHECK-NEXT: REPLICATE ir<%gep>.1 = getelementptr ir<%addr>, vp<[[STEPS_SUNK]]>1003; CHECK-NEXT: REPLICATE store ir<1.000000e+01>, ir<%gep>.11004; CHECK-NEXT: Successor(s): pred.store.continue1005; CHECK-EMPTY:1006; CHECK-NEXT: pred.store.continue:1007; CHECK-NEXT: No successors1008; CHECK-NEXT: }1009; CHECK-NEXT: Successor(s): then.01010; CHECK-EMPTY:1011; CHECK-NEXT: then.0:1012; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>1013; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>1014; CHECK-NEXT: No successors1015; CHECK-NEXT: }1016;1017entry:1018 br label %loop.header1019 1020loop.header:1021 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]1022 %gep = getelementptr float, ptr %addr, i64 %iv1023 %exitcond.not = icmp eq i64 %iv, 2001024 br i1 %exitcond.not, label %exit, label %loop.body1025 1026loop.body:1027 %0 = load float, ptr %gep, align 41028 %pred = fcmp oeq float %0, 0.01029 br i1 %pred, label %loop.latch, label %then1030 1031then:1032 store float 10.0, ptr %gep, align 41033 br label %loop.latch1034 1035loop.latch:1036 %iv.next = add nuw nsw i64 %iv, 11037 br label %loop.header1038 1039exit:1040 ret void1041}1042 1043; Test case with a dead GEP between the load and store regions. Dead recipes1044; need to be removed before merging.1045define void @merge_with_dead_gep_between_regions(i32 %n, i32 %k, ptr noalias %src, ptr noalias %dst) {1046; CHECK-LABEL: LV: Checking a loop in 'merge_with_dead_gep_between_regions'1047; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {1048; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF1049; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF1050; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count1051; CHECK-NEXT: Live-in ir<%n> = original trip-count1052; CHECK-EMPTY:1053; CHECK-NEXT: ir-bb<entry>:1054; CHECK-NEXT: Successor(s): scalar.ph, vector.ph1055; CHECK-EMPTY:1056; CHECK-NEXT: vector.ph:1057; CHECK-NEXT: vp<[[END:%.+]]> = DERIVED-IV ir<%n> + vp<[[VEC_TC]]> * ir<-1>1058; CHECK-NEXT: Successor(s): vector loop1059; CHECK-EMPTY:1060; CHECK-NEXT: <x1> vector loop: {1061; CHECK-NEXT: vector.body:1062; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION1063; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<%n>, ir<-1>, vp<[[VF]]>1064; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>1065; CHECK-NEXT: WIDEN ir<%cond> = icmp ult ir<%iv>, ir<%k>1066; CHECK-NEXT: Successor(s): pred.store1067; CHECK-EMPTY:1068; CHECK-NEXT: <xVFxUF> pred.store: {1069; CHECK-NEXT: pred.store.entry:1070; CHECK-NEXT: BRANCH-ON-MASK ir<%cond>1071; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue1072; CHECK-EMPTY:1073; CHECK-NEXT: pred.store.if:1074; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>, vp<[[VF]]>1075; CHECK-NEXT: REPLICATE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[SCALAR_STEPS]]>1076; CHECK-NEXT: REPLICATE ir<%l> = load ir<%gep.src>1077; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[SCALAR_STEPS]]>1078; CHECK-NEXT: REPLICATE store ir<%l>, ir<%gep.dst>1079; CHECK-NEXT: Successor(s): pred.store.continue1080; CHECK-EMPTY:1081; CHECK-NEXT: pred.store.continue:1082; CHECK-NEXT: No successors1083; CHECK-NEXT: }1084; CHECK-NEXT: Successor(s): loop.then.11085; CHECK-EMPTY:1086; CHECK-NEXT: loop.then.1:1087; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>1088; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>1089; CHECK-NEXT: No successors1090; CHECK-NEXT: }1091; CHECK-NEXT: Successor(s): middle.block1092; CHECK-EMPTY:1093; CHECK-NEXT: middle.block:1094; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<%n>, vp<[[VEC_TC]]>1095; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>1096; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph1097; CHECK-EMPTY:1098; CHECK-NEXT: ir-bb<exit>:1099; CHECK-NEXT: No successors1100; CHECK-EMPTY:1101; CHECK-NEXT: scalar.ph:1102; CHECK-NEXT: EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ vp<[[END]]>, middle.block ], [ ir<%n>, ir-bb<entry> ]1103; CHECK-NEXT: Successor(s): ir-bb<loop>1104; CHECK-EMPTY:1105; CHECK-NEXT: ir-bb<loop>:1106; CHECK-NEXT: IR %iv = phi i32 [ %n, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<[[RESUME]]> from scalar.ph)1107; CHECK-NEXT: IR %iv.next = add nsw i32 %iv, -11108; CHECK-NEXT: IR %cond = icmp ult i32 %iv, %k1109; CHECK-NEXT: No successors1110; CHECK-NEXT: }1111;1112entry:1113 br label %loop1114 1115loop:1116 %iv = phi i32[ %n, %entry ], [ %iv.next, %loop.latch ]1117 %iv.next = add nsw i32 %iv, -11118 %cond = icmp ult i32 %iv, %k1119 br i1 %cond, label %loop.then, label %loop.latch1120 1121loop.then:1122 %gep.src = getelementptr inbounds i32, ptr %src, i32 %iv1123 %l = load i32, ptr %gep.src, align 161124 %dead_gep = getelementptr inbounds i32, ptr %dst, i64 11125 %gep.dst = getelementptr inbounds i32, ptr %dst, i32 %iv1126 store i32 %l, ptr %gep.dst, align 161127 br label %loop.latch1128 1129loop.latch:1130 %ec = icmp eq i32 %iv.next, 01131 br i1 %ec, label %exit, label %loop1132 1133exit:1134 ret void1135}1136 1137define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {1138; CHECK-LABEL: LV: Checking a loop in 'ptr_induction_remove_dead_recipe'1139; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {1140; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF1141; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF1142; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count1143; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count1144; CHECK-EMPTY:1145; CHECK-NEXT: ir-bb<entry>:1146; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV ((-1 * (ptrtoint ptr %end to i64)) + (ptrtoint ptr %start to i64))1147; CHECK-NEXT: Successor(s): scalar.ph, vector.ph1148; CHECK-EMPTY:1149; CHECK-NEXT: vector.ph:1150; CHECK-NEXT: vp<[[END:%.+]]> = DERIVED-IV ir<%start> + vp<[[VEC_TC]]> * ir<-1>1151; CHECK-NEXT: Successor(s): vector loop1152; CHECK-EMPTY:1153; CHECK-NEXT: <x1> vector loop: {1154; CHECK-NEXT: vector.body:1155; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION1156; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<-1>1157; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>1158; CHECK-NEXT: EMIT vp<[[PTR_IV:%.+]]> = ptradd ir<%start>, vp<[[STEPS]]>1159; CHECK-NEXT: CLONE ir<%ptr.iv.next> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>1160; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-end-pointer inbounds ir<%ptr.iv.next>, vp<[[VF]]>1161; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]>1162; CHECK-NEXT: WIDEN ir<%c.1> = icmp ne ir<%l>, ir<0>1163; CHECK-NEXT: Successor(s): pred.store1164; CHECK-EMPTY:1165; CHECK-NEXT: <xVFxUF> pred.store: {1166; CHECK-NEXT: pred.store.entry:1167; CHECK-NEXT: BRANCH-ON-MASK ir<%c.1>1168; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue1169; CHECK-EMPTY:1170; CHECK-NEXT: pred.store.if:1171; CHECK-NEXT: REPLICATE ir<%ptr.iv.next>.1 = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>1172; CHECK-NEXT: REPLICATE store ir<95>, ir<%ptr.iv.next>.11173; CHECK-NEXT: Successor(s): pred.store.continue1174; CHECK-EMPTY:1175; CHECK-NEXT: pred.store.continue:1176; CHECK-NEXT: No successors1177; CHECK-NEXT: }1178; CHECK-NEXT: Successor(s): if.then.01179; CHECK-EMPTY:1180; CHECK-NEXT: if.then.0:1181; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>1182; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>1183; CHECK-NEXT: No successors1184; CHECK-NEXT: }1185; CHECK-NEXT: Successor(s): middle.block1186; CHECK-EMPTY:1187; CHECK-NEXT: middle.block:1188; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]>1189; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>1190; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph1191; CHECK-EMPTY:1192; CHECK-NEXT: ir-bb<exit>:1193; CHECK-NEXT: No successors1194; CHECK-EMPTY:1195; CHECK-NEXT: scalar.ph:1196; CHECK-NEXT: EMIT-SCALAR vp<[[RESUME:%.+]]> = phi [ vp<[[END]]>, middle.block ], [ ir<%start>, ir-bb<entry> ]1197; CHECK-NEXT: Successor(s): ir-bb<loop.header>1198; CHECK-EMPTY:1199; CHECK-NEXT: ir-bb<loop.header>:1200; CHECK-NEXT: IR %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ] (extra operand: vp<[[RESUME]]> from scalar.ph)1201; CHECK-NEXT: IR %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 -11202; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv.next, align 11203; CHECK-NEXT: IR %c.1 = icmp eq i8 %l, 01204; CHECK-NEXT: No successors1205; CHECK-NEXT: }1206;1207entry:1208 br label %loop.header1209 1210loop.header:1211 %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ]1212 %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 -11213 %l = load i8, ptr %ptr.iv.next, align 11214 %c.1 = icmp eq i8 %l, 01215 br i1 %c.1, label %loop.latch, label %if.then1216 1217if.then:1218 store i8 95, ptr %ptr.iv.next, align 11219 br label %loop.latch1220 1221loop.latch:1222 %c.2 = icmp eq ptr %ptr.iv.next, %end1223 br i1 %c.2, label %exit, label %loop.header1224 1225exit:1226 ret void1227}1228