222 lines · plain
1; REQUIRES: asserts2 3; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \4; RUN: -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue \5; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefixes=IF-EVL-OUTLOOP,IF-EVL %s6 7; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \8; RUN: -prefer-inloop-reductions \9; RUN: -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue \10; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefixes=IF-EVL-INLOOP,IF-EVL %s11 12; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \13; RUN: -prefer-predicate-over-epilogue=scalar-epilogue \14; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefixes=NO-VP-OUTLOOP %s15 16; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \17; RUN: -prefer-inloop-reductions \18; RUN: -prefer-predicate-over-epilogue=scalar-epilogue \19; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefixes=NO-VP-INLOOP %s20 21 22define i32 @reduction(ptr %a, i64 %n, i32 %start) {23; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'24; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI25 26; IF-EVL-OUTLOOP: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {27; IF-EVL-OUTLOOP-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF28; IF-EVL-OUTLOOP-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count29; IF-EVL-OUTLOOP-NEXT: Live-in ir<%n> = original trip-count30; IF-EVL-OUTLOOP-EMPTY:31; IF-EVL-OUTLOOP-NEXT: ir-bb<entry>:32; IF-EVL-OUTLOOP-NEXT: Successor(s): scalar.ph, vector.ph33; IF-EVL-OUTLOOP-EMPTY:34; IF-EVL-OUTLOOP-NEXT: vector.ph:35; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[RDX_START:%.]]> = reduction-start-vector ir<%start>, ir<0>, ir<1>36; IF-EVL-OUTLOOP-NEXT: Successor(s): vector loop37; IF-EVL-OUTLOOP-EMPTY:38; IF-EVL-OUTLOOP-NEXT: <x1> vector loop: {39; IF-EVL-OUTLOOP-NEXT: vector.body:40; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION41; IF-EVL-OUTLOOP-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>42; IF-EVL-OUTLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, vp<[[RDX_SELECT:%.+]]>43; IF-EVL-OUTLOOP-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ ir<%n>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]44; IF-EVL-OUTLOOP-NEXT: EMIT-SCALAR vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>45; IF-EVL-OUTLOOP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>46; IF-EVL-OUTLOOP-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>47; IF-EVL-OUTLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>48; IF-EVL-OUTLOOP-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>49; IF-EVL-OUTLOOP-NEXT: WIDEN ir<[[ADD:%.+]]> = add ir<[[LD1]]>, ir<[[RDX_PHI]]>50; IF-EVL-OUTLOOP-NEXT: WIDEN-INTRINSIC vp<[[RDX_SELECT]]> = call llvm.vp.merge(ir<true>, ir<[[ADD]]>, ir<[[RDX_PHI]]>, vp<[[EVL]]>)51; IF-EVL-OUTLOOP-NEXT: EMIT-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i6452; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>53; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[CAST]]>54; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>55; IF-EVL-OUTLOOP-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>56; IF-EVL-OUTLOOP-NEXT: No successors57; IF-EVL-OUTLOOP-NEXT: }58; IF-EVL-OUTLOOP-NEXT: Successor(s): middle.block59; IF-EVL-OUTLOOP-EMPTY:60; IF-EVL-OUTLOOP-NEXT: middle.block:61; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[RDX:%.+]]> = compute-reduction-result ir<[[RDX_PHI]]>, vp<[[RDX_SELECT]]>62; IF-EVL-OUTLOOP-NEXT: Successor(s): ir-bb<for.end>63; IF-EVL-OUTLOOP-EMPTY:64; IF-EVL-OUTLOOP-NEXT: ir-bb<for.end>:65; IF-EVL-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX]]> from middle.block)66; IF-EVL-OUTLOOP-NEXT: No successors67;68 69; IF-EVL-INLOOP: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {70; IF-EVL-INLOOP-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF71; IF-EVL-INLOOP-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count72; IF-EVL-INLOOP-NEXT: Live-in ir<%n> = original trip-count73; IF-EVL-INLOOP-EMPTY:74; IF-EVL-INLOOP: vector.ph:75; IF-EVL-INLOOP-NEXT: EMIT vp<[[RDX_START:%.]]> = reduction-start-vector ir<%start>, ir<0>, ir<1>76; IF-EVL-INLOOP-NEXT: Successor(s): vector loop77; IF-EVL-INLOOP-EMPTY:78; IF-EVL-INLOOP-NEXT: <x1> vector loop: {79; IF-EVL-INLOOP-NEXT: vector.body:80; IF-EVL-INLOOP-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION81; IF-EVL-INLOOP-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>82; IF-EVL-INLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>83; IF-EVL-INLOOP-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ ir<%n>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]84; IF-EVL-INLOOP-NEXT: EMIT-SCALAR vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>85; IF-EVL-INLOOP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>86; IF-EVL-INLOOP-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>87; IF-EVL-INLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>88; IF-EVL-INLOOP-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>89; IF-EVL-INLOOP-NEXT: REDUCE ir<[[ADD:%.+]]> = ir<[[RDX_PHI]]> + vp.reduce.add (ir<[[LD1]]>, vp<[[EVL]]>)90; IF-EVL-INLOOP-NEXT: EMIT-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i6491; IF-EVL-INLOOP-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>92; IF-EVL-INLOOP-NEXT: EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[CAST]]>93; IF-EVL-INLOOP-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>94; IF-EVL-INLOOP-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>95; IF-EVL-INLOOP-NEXT: No successors96; IF-EVL-INLOOP-NEXT: }97; IF-EVL-INLOOP-NEXT: Successor(s): middle.block98; IF-EVL-INLOOP-EMPTY:99; IF-EVL-INLOOP-NEXT: middle.block:100; IF-EVL-INLOOP-NEXT: EMIT vp<[[RDX:%.+]]> = compute-reduction-result ir<[[RDX_PHI]]>, ir<[[ADD]]>101; IF-EVL-INLOOP-NEXT: Successor(s): ir-bb<for.end>102; IF-EVL-INLOOP-EMPTY:103; IF-EVL-INLOOP-NEXT: ir-bb<for.end>:104; IF-EVL-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX]]> from middle.block)105; IF-EVL-INLOOP-NEXT: No successors106;107 108; NO-VP-OUTLOOP: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF>=1' {109; NO-VP-OUTLOOP-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF110; NO-VP-OUTLOOP-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF111; NO-VP-OUTLOOP-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count112; NO-VP-OUTLOOP-NEXT: Live-in ir<%n> = original trip-count113; NO-VP-OUTLOOP-EMPTY:114; NO-VP-OUTLOOP: vector.ph:115; NO-VP-OUTLOOP-NEXT: EMIT vp<[[RDX_START:%.]]> = reduction-start-vector ir<%start>, ir<0>, ir<1>116; NO-VP-OUTLOOP-NEXT: Successor(s): vector loop117; NO-VP-OUTLOOP-EMPTY:118; NO-VP-OUTLOOP-NEXT: <x1> vector loop: {119; NO-VP-OUTLOOP-NEXT: vector.body:120; NO-VP-OUTLOOP-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION121; NO-VP-OUTLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>122; NO-VP-OUTLOOP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>, vp<[[VF]]>123; NO-VP-OUTLOOP-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>124; NO-VP-OUTLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>125; NO-VP-OUTLOOP-NEXT: WIDEN ir<[[LD1:%.+]]> = load vp<[[PTR1]]>126; NO-VP-OUTLOOP-NEXT: WIDEN ir<[[ADD:%.+]]> = add ir<[[LD1]]>, ir<[[RDX_PHI]]>127; NO-VP-OUTLOOP-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add nuw vp<[[IV]]>, vp<[[VFUF]]>128; NO-VP-OUTLOOP-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>129; NO-VP-OUTLOOP-NEXT: No successors130; NO-VP-OUTLOOP-NEXT: }131; NO-VP-OUTLOOP-NEXT: Successor(s): middle.block132; NO-VP-OUTLOOP-EMPTY:133; NO-VP-OUTLOOP-NEXT: middle.block:134; NO-VP-OUTLOOP-NEXT: EMIT vp<[[RDX:%.+]]> = compute-reduction-result ir<[[RDX_PHI]]>, ir<[[ADD]]>135; NO-VP-OUTLOOP-NEXT: EMIT vp<[[BOC:%.+]]> = icmp eq ir<%n>, vp<[[VTC]]>136; NO-VP-OUTLOOP-NEXT: EMIT branch-on-cond vp<[[BOC]]>137; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph138; NO-VP-OUTLOOP-EMPTY:139; NO-VP-OUTLOOP-NEXT: ir-bb<for.end>:140; NO-VP-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX]]> from middle.block)141; NO-VP-OUTLOOP-NEXT: No successors142; NO-VP-OUTLOOP-EMPTY:143; NO-VP-OUTLOOP-NEXT: scalar.ph:144; NO-VP-OUTLOOP-NEXT: EMIT-SCALAR vp<[[IV_RESUME:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]145; NO-VP-OUTLOOP-NEXT: EMIT-SCALAR vp<[[RED_RESUME:%.+]]> = phi [ vp<[[RDX]]>, middle.block ], [ ir<%start>, ir-bb<entry> ]146; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb<for.body>147; NO-VP-OUTLOOP-EMPTY:148; NO-VP-OUTLOOP-NEXT: ir-bb<for.body>:149; NO-VP-OUTLOOP-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] (extra operand: vp<[[IV_RESUME]]> from scalar.ph)150; NO-VP-OUTLOOP-NEXT: IR %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ]151; NO-VP-OUTLOOP: IR %exitcond.not = icmp eq i64 %iv.next, %n152; NO-VP-OUTLOOP-NEXT: No successors153; NO-VP-OUTLOOP-NEXT: }154;155 156; NO-VP-INLOOP: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF>=1' {157; NO-VP-INLOOP-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF158; NO-VP-INLOOP-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF159; NO-VP-INLOOP-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count160; NO-VP-INLOOP-NEXT: Live-in ir<%n> = original trip-count161; NO-VP-INLOOP-EMPTY:162; NO-VP-INLOOP: vector.ph:163; NO-VP-INLOOP-NEXT: EMIT vp<[[RDX_START:%.]]> = reduction-start-vector ir<%start>, ir<0>, ir<1>164; NO-VP-INLOOP-NEXT: Successor(s): vector loop165; NO-VP-INLOOP-EMPTY:166; NO-VP-INLOOP-NEXT: <x1> vector loop: {167; NO-VP-INLOOP-NEXT: vector.body:168; NO-VP-INLOOP-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION169; NO-VP-INLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>170; NO-VP-INLOOP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>, vp<[[VF]]>171; NO-VP-INLOOP-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>172; NO-VP-INLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>173; NO-VP-INLOOP-NEXT: WIDEN ir<[[LD1:%.+]]> = load vp<[[PTR1]]>174; NO-VP-INLOOP-NEXT: REDUCE ir<[[ADD:%.+]]> = ir<[[RDX_PHI]]> + reduce.add (ir<[[LD1]]>)175; NO-VP-INLOOP-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add nuw vp<[[IV]]>, vp<[[VFUF]]>176; NO-VP-INLOOP-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>177; NO-VP-INLOOP-NEXT: No successors178; NO-VP-INLOOP-NEXT: }179; NO-VP-INLOOP-NEXT: Successor(s): middle.block180; NO-VP-INLOOP-EMPTY:181; NO-VP-INLOOP-NEXT: middle.block:182; NO-VP-INLOOP-NEXT: EMIT vp<[[RDX:%.+]]> = compute-reduction-result ir<[[RDX_PHI]]>, ir<[[ADD]]>183; NO-VP-INLOOP-NEXT: EMIT vp<[[BOC:%.+]]> = icmp eq ir<%n>, vp<[[VTC]]>184; NO-VP-INLOOP-NEXT: EMIT branch-on-cond vp<[[BOC]]>185; NO-VP-INLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph186; NO-VP-INLOOP-EMPTY:187; NO-VP-INLOOP-NEXT: ir-bb<for.end>:188; NO-VP-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX]]> from middle.block)189; NO-VP-INLOOP-NEXT: No successors190; NO-VP-INLOOP-EMPTY:191; NO-VP-INLOOP-NEXT: scalar.ph:192; NO-VP-INLOOP-NEXT: EMIT-SCALAR vp<[[IV_RESUME:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb<entry> ]193; NO-VP-INLOOP-NEXT: EMIT-SCALAR vp<[[RED_RESUME:%.+]]> = phi [ vp<[[RDX]]>, middle.block ], [ ir<%start>, ir-bb<entry> ]194; NO-VP-INLOOP-NEXT: Successor(s): ir-bb<for.body>195; NO-VP-INLOOP-EMPTY:196; NO-VP-INLOOP-NEXT: ir-bb<for.body>:197; NO-VP-INLOOP-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] (extra operand: vp<[[IV_RESUME]]> from scalar.ph)198; NO-VP-INLOOP-NEXT: IR %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ]199; NO-VP-INLOOP: IR %exitcond.not = icmp eq i64 %iv.next, %n200; NO-VP-INLOOP-NEXT: No successors201; NO-VP-INLOOP-NEXT: }202;203entry:204 br label %for.body205 206for.body:207 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]208 %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ]209 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv210 %0 = load i32, ptr %arrayidx, align 4211 %add = add nsw i32 %0, %rdx212 %iv.next = add nuw nsw i64 %iv, 1213 %exitcond.not = icmp eq i64 %iv.next, %n214 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0215 216for.end:217 ret i32 %add218}219 220!0 = distinct !{!0, !1}221!1 = !{!"llvm.loop.vectorize.enable", i1 true}222