286 lines · plain
1; REQUIRES: asserts2; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck %s3 4target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"5target triple = "aarch64-none-unknown-elf"6 7; Tests for printing VPlans that are enabled under AArch648 9define i32 @print_partial_reduction(ptr %a, ptr %b) "target-features"="+neon,+dotprod" {10; CHECK: VPlan 'Initial VPlan for VF={8,16},UF>=1' {11; CHECK-NEXT: Live-in vp<[[VF:%.]]> = VF12; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF13; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count14; CHECK-NEXT: Live-in ir<1024> = original trip-count15; CHECK-EMPTY:16; CHECK-NEXT: ir-bb<entry>:17; CHECK-NEXT: Successor(s): scalar.ph, vector.ph18; CHECK-EMPTY:19; CHECK-NEXT: vector.ph:20; CHECK-NEXT: EMIT vp<[[RDX_START:%.+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>21; CHECK-NEXT: Successor(s): vector loop22; CHECK-EMPTY:23; CHECK-NEXT: <x1> vector loop: {24; CHECK-NEXT: vector.body:25; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>26; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<[[ACC:%.+]]> = phi vp<[[RDX_START]]>, vp<[[REDUCE:%.+]]> (VF scaled by 1/4)27; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>, vp<[[VF]]>28; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[STEPS]]>29; CHECK-NEXT: vp<[[PTR_A:%.+]]> = vector-pointer ir<%gep.a>30; CHECK-NEXT: WIDEN ir<%load.a> = load vp<[[PTR_A]]>31; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<[[STEPS]]>32; CHECK-NEXT: vp<[[PTR_B:%.+]]> = vector-pointer ir<%gep.b>33; CHECK-NEXT: WIDEN ir<%load.b> = load vp<[[PTR_B]]>34; CHECK-NEXT: EXPRESSION vp<[[REDUCE]]> = ir<[[ACC]]> + partial.reduce.add (mul (ir<%load.b> zext to i32), (ir<%load.a> zext to i32))35; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>36; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>37; CHECK-NEXT: No successors38; CHECK-NEXT: }39; CHECK-NEXT: Successor(s): middle.block40; CHECK-EMPTY:41; CHECK-NEXT: middle.block:42; CHECK-NEXT: EMIT vp<[[RED_RESULT:%.+]]> = compute-reduction-result ir<[[ACC]]>, vp<[[REDUCE]]>43; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, vp<[[VEC_TC]]>44; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>45; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph46; CHECK-EMPTY:47; CHECK-NEXT: ir-bb<exit>:48; CHECK-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RED_RESULT]]> from middle.block)49; CHECK-NEXT: No successors50; CHECK-EMPTY:51; CHECK-NEXT: scalar.ph:52; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VEC_TC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]53; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[RED_RESULT]]>, middle.block ], [ ir<0>, ir-bb<entry> ]54; CHECK-NEXT: Successor(s): ir-bb<for.body>55; CHECK-EMPTY:56; CHECK-NEXT: ir-bb<for.body>:57; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]58; CHECK-NEXT: IR %accum = phi i32 [ 0, %entry ], [ %add, %for.body ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)59; CHECK-NEXT: IR %gep.a = getelementptr i8, ptr %a, i64 %iv60; CHECK-NEXT: IR %load.a = load i8, ptr %gep.a, align 161; CHECK-NEXT: IR %ext.a = zext i8 %load.a to i3262; CHECK-NEXT: IR %gep.b = getelementptr i8, ptr %b, i64 %iv63; CHECK-NEXT: IR %load.b = load i8, ptr %gep.b, align 164; CHECK-NEXT: IR %ext.b = zext i8 %load.b to i3265; CHECK-NEXT: IR %mul = mul i32 %ext.b, %ext.a66; CHECK-NEXT: IR %add = add i32 %mul, %accum67; CHECK-NEXT: IR %iv.next = add i64 %iv, 168; CHECK-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, 102469; CHECK-NEXT: No successors70; CHECK-NEXT: }71; CHECK: VPlan 'Final VPlan for VF={8,16},UF={1}' {72; CHECK-NEXT: Live-in ir<1024> = vector-trip-count73; CHECK-NEXT: Live-in ir<1024> = original trip-count74; CHECK-EMPTY:75; CHECK-NEXT: ir-bb<entry>:76; CHECK-NEXT: Successor(s): vector.ph77; CHECK-EMPTY:78; CHECK-NEXT: vector.ph:79; CHECK-NEXT: EMIT vp<[[RDX_START:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>80; CHECK-NEXT: Successor(s): vector.body81; CHECK-EMPTY:82; CHECK-NEXT: vector.body:83; CHECK-NEXT: EMIT-SCALAR vp<[[EP_IV:%.+]]> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]84; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]> (VF scaled by 1/4)85; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<%index>86; CHECK-NEXT: WIDEN ir<%load.a> = load ir<%gep.a>87; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<%index>88; CHECK-NEXT: WIDEN ir<%load.b> = load ir<%gep.b>89; CHECK-NEXT: WIDEN-CAST ir<%ext.b> = zext ir<%load.b> to i3290; CHECK-NEXT: WIDEN-CAST ir<%ext.a> = zext ir<%load.a> to i3291; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%ext.b>, ir<%ext.a>92; CHECK-NEXT: PARTIAL-REDUCE ir<[[RDX_NEXT]]> = ir<[[RDX]]> + reduce.add (ir<%mul>)93; CHECK-NEXT: EMIT vp<[[EP_IV_NEXT:%.+]]> = add nuw vp<[[EP_IV]]>, ir<16>94; CHECK-NEXT: EMIT branch-on-count vp<[[EP_IV_NEXT]]>, ir<1024>95; CHECK-NEXT: Successor(s): middle.block, vector.body96; CHECK-EMPTY:97; CHECK-NEXT: middle.block:98; CHECK-NEXT: EMIT vp<[[RED_RESULT:%[0-9]+]]> = compute-reduction-result ir<[[RDX]]>, ir<[[RDX_NEXT]]>99; CHECK-NEXT: Successor(s): ir-bb<exit>100; CHECK-EMPTY:101; CHECK-NEXT: ir-bb<exit>:102; CHECK-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RED_RESULT]]> from middle.block)103; CHECK-NEXT: No successors104; CHECK-NEXT: }105entry:106 br label %for.body107 108for.body: ; preds = %for.body, %entry109 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]110 %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]111 %gep.a = getelementptr i8, ptr %a, i64 %iv112 %load.a = load i8, ptr %gep.a, align 1113 %ext.a = zext i8 %load.a to i32114 %gep.b = getelementptr i8, ptr %b, i64 %iv115 %load.b = load i8, ptr %gep.b, align 1116 %ext.b = zext i8 %load.b to i32117 %mul = mul i32 %ext.b, %ext.a118 %add = add i32 %mul, %accum119 %iv.next = add i64 %iv, 1120 %exitcond.not = icmp eq i64 %iv.next, 1024121 br i1 %exitcond.not, label %exit, label %for.body, !llvm.loop !0122 123exit:124 ret i32 %add125}126 127; Test that we also get VPExpressions when there is predication.128define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target-features"="+sve" {129; CHECK: VPlan 'Initial VPlan for VF={8,16},UF>=1' {130; CHECK-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF131; CHECK-NEXT: Live-in vp<[[VFxUF:%[0-9]+]]> = VF * UF132; CHECK-NEXT: Live-in ir<%N> = original trip-count133; CHECK-EMPTY:134; CHECK-NEXT: ir-bb<entry>:135; CHECK-NEXT: Successor(s): scalar.ph, vector.ph136; CHECK-EMPTY:137; CHECK-NEXT: vector.ph:138; CHECK-NEXT: EMIT vp<[[RDX_START:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>139; CHECK-NEXT: EMIT vp<[[TC_MINUS_VF:%[0-9]+]]> = TC > VF ? TC - VF : 0 ir<%N>140; CHECK-NEXT: EMIT vp<%index.part.next> = VF * Part + ir<0>141; CHECK-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask vp<%index.part.next>, ir<%N>, ir<1>142; CHECK-NEXT: Successor(s): vector loop143; CHECK-EMPTY:144; CHECK-NEXT: <x1> vector loop: {145; CHECK-NEXT: vector.body:146; CHECK-NEXT: EMIT vp<[[CAN_IV:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>147; CHECK-NEXT: ACTIVE-LANE-MASK-PHI vp<[[MASK:%[0-9]+]]> = phi vp<%active.lane.mask.entry>, vp<%active.lane.mask.next>148; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi vp<[[RDX_START]]>, vp<[[REDUCE:%[0-9]+]]> (VF scaled by 1/4)149; CHECK-NEXT: vp<[[STEPS:%[0-9]+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>, vp<[[VF]]>150; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[STEPS]]>151; CHECK-NEXT: vp<[[PTR_A:%[0-9]+]]> = vector-pointer ir<%gep.a>152; CHECK-NEXT: WIDEN ir<%load.a> = load vp<[[PTR_A]]>, vp<[[MASK]]>153; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<[[STEPS]]>154; CHECK-NEXT: vp<[[PTR_B:%[0-9]+]]> = vector-pointer ir<%gep.b>155; CHECK-NEXT: WIDEN ir<%load.b> = load vp<[[PTR_B]]>, vp<[[MASK]]>156; CHECK-NEXT: EXPRESSION vp<[[REDUCE]]> = vp<[[MASK]]> + partial.reduce.add (mul (ir<%load.b> zext to i32), (ir<%load.a> zext to i32), <badref>)157; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]>158; CHECK-NEXT: EMIT vp<[[PART_IDX:%[0-9]+]]> = VF * Part + vp<[[CAN_IV]]>159; CHECK-NEXT: EMIT vp<%active.lane.mask.next> = active lane mask vp<[[PART_IDX]]>, vp<[[TC_MINUS_VF]]>, ir<1>160; CHECK-NEXT: EMIT vp<[[NOT_MASK:%[0-9]+]]> = not vp<%active.lane.mask.next>161; CHECK-NEXT: EMIT branch-on-cond vp<[[NOT_MASK]]>162; CHECK-NEXT: No successors163; CHECK-NEXT: }164; CHECK-NEXT: Successor(s): middle.block165; CHECK-EMPTY:166; CHECK-NEXT: middle.block:167; CHECK-NEXT: EMIT vp<[[RED_RESULT:%[0-9]+]]> = compute-reduction-result ir<%accum>, vp<[[REDUCE]]>168; CHECK-NEXT: Successor(s): ir-bb<exit>169; CHECK-EMPTY:170; CHECK-NEXT: ir-bb<exit>:171; CHECK-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RED_RESULT]]> from middle.block)172; CHECK-NEXT: No successors173entry:174 br label %for.body175 176for.body: ; preds = %for.body, %entry177 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]178 %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]179 %gep.a = getelementptr i8, ptr %a, i64 %iv180 %load.a = load i8, ptr %gep.a, align 1181 %ext.a = zext i8 %load.a to i32182 %gep.b = getelementptr i8, ptr %b, i64 %iv183 %load.b = load i8, ptr %gep.b, align 1184 %ext.b = zext i8 %load.b to i32185 %mul = mul i32 %ext.b, %ext.a186 %add = add i32 %mul, %accum187 %iv.next = add i64 %iv, 1188 %exitcond.not = icmp eq i64 %iv.next, %N189 br i1 %exitcond.not, label %exit, label %for.body, !llvm.loop !1190 191exit:192 ret i32 %add193}194 195define i32 @print_partial_reduction_ext_mul(ptr %a, ptr %b) "target-features"="+neon,+dotprod" {196; CHECK: VPlan 'Initial VPlan for VF={8,16},UF>=1' {197; CHECK-NEXT: Live-in vp<[[VF:%.]]> = VF198; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF199; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count200; CHECK-NEXT: Live-in ir<1024> = original trip-count201; CHECK-EMPTY:202; CHECK-NEXT: ir-bb<entry>:203; CHECK-NEXT: Successor(s): scalar.ph, vector.ph204; CHECK-EMPTY:205; CHECK-NEXT: vector.ph:206; CHECK-NEXT: EMIT vp<[[RDX_START:%.+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>207; CHECK-NEXT: Successor(s): vector loop208; CHECK-EMPTY:209; CHECK-NEXT: <x1> vector loop: {210; CHECK-NEXT: vector.body:211; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>212; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<[[ACC:%.+]]> = phi vp<[[RDX_START]]>, vp<[[REDUCE:%.+]]> (VF scaled by 1/4)213; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>, vp<[[VF]]>214; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[STEPS]]>215; CHECK-NEXT: vp<[[PTR_A:%.+]]> = vector-pointer ir<%gep.a>216; CHECK-NEXT: WIDEN ir<%load.a> = load vp<[[PTR_A]]>217; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<[[STEPS]]>218; CHECK-NEXT: vp<[[PTR_B:%.+]]> = vector-pointer ir<%gep.b>219; CHECK-NEXT: WIDEN ir<%load.b> = load vp<[[PTR_B]]>220; CHECK-NEXT: EXPRESSION vp<[[REDUCE]]> = ir<[[ACC]]> + partial.reduce.add (mul (ir<%load.b> zext to i32), (ir<%load.a> zext to i32))221; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>222; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>223; CHECK-NEXT: No successors224; CHECK-NEXT: }225; CHECK-NEXT: Successor(s): middle.block226; CHECK-EMPTY:227; CHECK-NEXT: middle.block:228; CHECK-NEXT: EMIT vp<[[RED_RESULT:%.+]]> = compute-reduction-result ir<[[ACC]]>, vp<[[REDUCE]]>229; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, vp<[[VEC_TC]]>230; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>231; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph232; CHECK-EMPTY:233; CHECK-NEXT: ir-bb<exit>:234; CHECK-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RED_RESULT]]> from middle.block)235; CHECK-NEXT: No successors236; CHECK-EMPTY:237; CHECK-NEXT: scalar.ph:238; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VEC_TC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]239; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[RED_RESULT]]>, middle.block ], [ ir<0>, ir-bb<entry> ]240; CHECK-NEXT: Successor(s): ir-bb<for.body>241; CHECK-EMPTY:242; CHECK-NEXT: ir-bb<for.body>:243; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]244; CHECK-NEXT: IR %accum = phi i32 [ 0, %entry ], [ %add, %for.body ] (extra operand: vp<%bc.merge.rdx> from scalar.ph)245; CHECK-NEXT: IR %gep.a = getelementptr i8, ptr %a, i64 %iv246; CHECK-NEXT: IR %load.a = load i8, ptr %gep.a, align 1247; CHECK-NEXT: IR %ext.a = zext i8 %load.a to i16248; CHECK-NEXT: IR %gep.b = getelementptr i8, ptr %b, i64 %iv249; CHECK-NEXT: IR %load.b = load i8, ptr %gep.b, align 1250; CHECK-NEXT: IR %ext.b = zext i8 %load.b to i16251; CHECK-NEXT: IR %mul = mul i16 %ext.b, %ext.a252; CHECK-NEXT: IR %mul.ext = zext i16 %mul to i32253; CHECK-NEXT: IR %add = add i32 %mul.ext, %accum254; CHECK-NEXT: IR %iv.next = add i64 %iv, 1255; CHECK-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, 1024256; CHECK-NEXT: No successors257; CHECK-NEXT: }258entry:259 br label %for.body260 261for.body: ; preds = %for.body, %entry262 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]263 %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]264 %gep.a = getelementptr i8, ptr %a, i64 %iv265 %load.a = load i8, ptr %gep.a, align 1266 %ext.a = zext i8 %load.a to i16267 %gep.b = getelementptr i8, ptr %b, i64 %iv268 %load.b = load i8, ptr %gep.b, align 1269 %ext.b = zext i8 %load.b to i16270 %mul = mul i16 %ext.b, %ext.a271 %mul.ext = zext i16 %mul to i32272 %add = add i32 %mul.ext, %accum273 %iv.next = add i64 %iv, 1274 %exitcond.not = icmp eq i64 %iv.next, 1024275 br i1 %exitcond.not, label %exit, label %for.body276 277exit:278 ret i32 %add279}280 281!0 = distinct !{!0, !2, !3}282!1 = distinct !{!1, !2, !4}283!2 = !{!"llvm.loop.interleave.count", i32 1}284!3 = !{!"llvm.loop.vectorize.predicate.enable", i1 false}285!4 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}286