205 lines · plain
1; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S \2; RUN: -prefer-predicate-over-epilogue=scalar-epilogue < %s | FileCheck %s --check-prefix=CHECK-VF4IC13; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -S \4; RUN: -prefer-predicate-over-epilogue=scalar-epilogue < %s | FileCheck %s --check-prefix=CHECK-VF4IC45 6target triple = "aarch64-linux-gnu"7 8define i32 @select_const_i32_from_icmp(ptr nocapture readonly %v, i64 %n) #0 {9; CHECK-VF4IC1-LABEL: @select_const_i32_from_icmp10; CHECK-VF4IC1: vector.body:11; CHECK-VF4IC1: [[VEC_PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL:%.*]], %vector.body ]12; CHECK-VF4IC1: [[VEC_LOAD:%.*]] = load <vscale x 4 x i32>13; CHECK-VF4IC1-NEXT: [[VEC_ICMP:%.*]] = icmp ne <vscale x 4 x i32> [[VEC_LOAD]], splat (i32 3)14; CHECK-VF4IC1-NEXT: [[VEC_SEL]] = or <vscale x 4 x i1> [[VEC_PHI]], [[VEC_ICMP]]15; CHECK-VF4IC1: middle.block:16; CHECK-VF4IC1-NEXT: [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[VEC_SEL]])17; CHECK-VF4IC1-NEXT: [[FR:%.*]] = freeze i1 [[OR_RDX]]18; CHECK-VF4IC1-NEXT: {{.*}} = select i1 [[FR]], i32 7, i32 319; CHECK-VF4IC1: %cmp.n = icmp eq i64 %n, %n.vec20 21; CHECK-VF4IC4-LABEL: @select_const_i32_from_icmp22; CHECK-VF4IC4: vector.body:23; CHECK-VF4IC4: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL1:%.*]], %vector.body ]24; CHECK-VF4IC4: [[VEC_PHI2:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL2:%.*]], %vector.body ]25; CHECK-VF4IC4: [[VEC_PHI3:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL3:%.*]], %vector.body ]26; CHECK-VF4IC4: [[VEC_PHI4:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL4:%.*]], %vector.body ]27; CHECK-VF4IC4: [[VEC_ICMP1:%.*]] = icmp ne <vscale x 4 x i32> {{.*}}, splat (i32 3)28; CHECK-VF4IC4-NEXT: [[VEC_ICMP2:%.*]] = icmp ne <vscale x 4 x i32> {{.*}}, splat (i32 3)29; CHECK-VF4IC4-NEXT: [[VEC_ICMP3:%.*]] = icmp ne <vscale x 4 x i32> {{.*}}, splat (i32 3)30; CHECK-VF4IC4-NEXT: [[VEC_ICMP4:%.*]] = icmp ne <vscale x 4 x i32> {{.*}}, splat (i32 3)31; CHECK-VF4IC4-NEXT: [[VEC_SEL1:%.*]] = or <vscale x 4 x i1> [[VEC_PHI1]], [[VEC_ICMP1]]32; CHECK-VF4IC4-NEXT: [[VEC_SEL2:%.*]] = or <vscale x 4 x i1> [[VEC_PHI2]], [[VEC_ICMP2]]33; CHECK-VF4IC4-NEXT: [[VEC_SEL3:%.*]] = or <vscale x 4 x i1> [[VEC_PHI3]], [[VEC_ICMP3]]34; CHECK-VF4IC4-NEXT: [[VEC_SEL4:%.*]] = or <vscale x 4 x i1> [[VEC_PHI4]], [[VEC_ICMP4]]35; CHECK-VF4IC4: middle.block:36; CHECK-VF4IC4-NEXT: [[OR1:%.*]] = or <vscale x 4 x i1> [[VEC_SEL2]], [[VEC_SEL1]]37; CHECK-VF4IC4-NEXT: [[OR2:%.*]] = or <vscale x 4 x i1> [[VEC_SEL3]], [[OR1]]38; CHECK-VF4IC4-NEXT: [[OR3:%.*]] = or <vscale x 4 x i1> [[VEC_SEL4]], [[OR2]]39; CHECK-VF4IC4-NEXT: [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[OR3]])40; CHECK-VF4IC4-NEXT: [[FR:%.*]] = freeze i1 [[OR_RDX]]41; CHECK-VF4IC4-NEXT: {{.*}} = select i1 [[FR]], i32 7, i32 342; CHECK-VF4IC4-NEXT: %cmp.n = icmp eq i64 %n, %n.vec43entry:44 br label %for.body45 46for.body: ; preds = %entry, %for.body47 %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]48 %1 = phi i32 [ 3, %entry ], [ %5, %for.body ]49 %2 = getelementptr inbounds i32, ptr %v, i64 %050 %3 = load i32, ptr %2, align 451 %4 = icmp eq i32 %3, 352 %5 = select i1 %4, i32 %1, i32 753 %6 = add nuw nsw i64 %0, 154 %7 = icmp eq i64 %6, %n55 br i1 %7, label %exit, label %for.body, !llvm.loop !056 57exit: ; preds = %for.body58 ret i32 %559}60 61define i32 @select_i32_from_icmp(ptr nocapture readonly %v, i32 %a, i32 %b, i64 %n) #0 {62; CHECK-VF4IC1-LABEL: @select_i32_from_icmp63; CHECK-VF4IC1: vector.ph:64; CHECK-VF4IC1-NOT: shufflevector <vscale x 4 x i32>65; CHECK-VF4IC1-NOT: shufflevector <vscale x 4 x i32>66; CHECK-VF4IC1: vector.body:67; CHECK-VF4IC1: [[VEC_PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL:%.*]], %vector.body ]68; CHECK-VF4IC1: [[VEC_LOAD:%.*]] = load <vscale x 4 x i32>69; CHECK-VF4IC1-NEXT: [[VEC_ICMP:%.*]] = icmp ne <vscale x 4 x i32> [[VEC_LOAD]], splat (i32 3)70; CHECK-VF4IC1-NEXT: [[VEC_SEL]] = or <vscale x 4 x i1> [[VEC_PHI]], [[VEC_ICMP]]71; CHECK-VF4IC1: middle.block:72; CHECK-VF4IC1-NEXT: [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[VEC_SEL]])73; CHECK-VF4IC1-NEXT: [[FR:%.*]] = freeze i1 [[OR_RDX]]74; CHECK-VF4IC1-NEXT: {{.*}} = select i1 [[FR]], i32 %b, i32 %a75; CHECK-VF4IC1: %cmp.n = icmp eq i64 %n, %n.vec76 77; CHECK-VF4IC4-LABEL: @select_i32_from_icmp78; CHECK-VF4IC4: vector.body:79entry:80 br label %for.body81 82for.body: ; preds = %entry, %for.body83 %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]84 %1 = phi i32 [ %a, %entry ], [ %5, %for.body ]85 %2 = getelementptr inbounds i32, ptr %v, i64 %086 %3 = load i32, ptr %2, align 487 %4 = icmp eq i32 %3, 388 %5 = select i1 %4, i32 %1, i32 %b89 %6 = add nuw nsw i64 %0, 190 %7 = icmp eq i64 %6, %n91 br i1 %7, label %exit, label %for.body, !llvm.loop !092 93exit: ; preds = %for.body94 ret i32 %595}96 97define i32 @select_const_i32_from_fcmp(ptr nocapture readonly %v, i64 %n) #0 {98; CHECK-VF4IC1-LABEL: @select_const_i32_from_fcmp99; CHECK-VF4IC1: vector.body:100; CHECK-VF4IC1: [[VEC_PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL:%.*]], %vector.body ]101; CHECK-VF4IC1: [[VEC_LOAD:%.*]] = load <vscale x 4 x float>102; CHECK-VF4IC1-NEXT: [[VEC_ICMP:%.*]] = fcmp fast one <vscale x 4 x float> [[VEC_LOAD]], splat (float 3.000000e+00)103; CHECK-VF4IC1-NEXT: [[VEC_SEL]] = or <vscale x 4 x i1> [[VEC_PHI]], [[VEC_ICMP]]104; CHECK-VF4IC1: middle.block:105; CHECK-VF4IC1-NEXT: [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[VEC_SEL]])106; CHECK-VF4IC1-NEXT: [[FR:%.*]] = freeze i1 [[OR_RDX]]107; CHECK-VF4IC1-NEXT: {{.*}} = select i1 [[FR]], i32 1, i32 2108; CHECK-VF4IC1: %cmp.n = icmp eq i64 %n, %n.vec109 110; CHECK-VF4IC4-LABEL: @select_const_i32_from_fcmp111; CHECK-VF4IC4: vector.body:112entry:113 br label %for.body114 115for.body: ; preds = %entry, %for.body116 %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]117 %1 = phi i32 [ 2, %entry ], [ %5, %for.body ]118 %2 = getelementptr inbounds float, ptr %v, i64 %0119 %3 = load float, ptr %2, align 4120 %4 = fcmp fast ueq float %3, 3.0121 %5 = select i1 %4, i32 %1, i32 1122 %6 = add nuw nsw i64 %0, 1123 %7 = icmp eq i64 %6, %n124 br i1 %7, label %exit, label %for.body, !llvm.loop !0125 126exit: ; preds = %for.body127 ret i32 %5128}129 130define float @select_const_f32_from_icmp(ptr nocapture readonly %v, i64 %n) #0 {131; CHECK-VF4IC1-LABEL: @select_const_f32_from_icmp132; CHECK-VF4IC1-NOT: vector.body133; CHECK-VF4IC4-LABEL: @select_const_f32_from_icmp134; CHECK-VF4IC4-NOT: vector.body135entry:136 br label %for.body137 138for.body: ; preds = %entry, %for.body139 %0 = phi i64 [ 0, %entry ], [ %6, %for.body ]140 %1 = phi fast float [ 3.0, %entry ], [ %5, %for.body ]141 %2 = getelementptr inbounds i32, ptr %v, i64 %0142 %3 = load i32, ptr %2, align 4143 %4 = icmp eq i32 %3, 3144 %5 = select fast i1 %4, float %1, float 7.0145 %6 = add nuw nsw i64 %0, 1146 %7 = icmp eq i64 %6, %n147 br i1 %7, label %exit, label %for.body, !llvm.loop !0148 149exit: ; preds = %for.body150 ret float %5151}152 153define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i64 %n) #0 {154; CHECK-VF4IC1-LABEL: @pred_select_const_i32_from_icmp155; CHECK-VF4IC1: vector.body:156; CHECK-VF4IC1: [[VEC_PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %vector.ph ], [ [[VEC_SEL:%.*]], %vector.body ]157; CHECK-VF4IC1: [[VEC_LOAD:%.*]] = load <vscale x 4 x i32>158; CHECK-VF4IC1: [[MASK:%.*]] = icmp sgt <vscale x 4 x i32> [[VEC_LOAD]], splat (i32 35)159; CHECK-VF4IC1: [[MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 {{%.*}}, <vscale x 4 x i1> [[MASK]], <vscale x 4 x i32> poison)160; CHECK-VF4IC1-NEXT: [[VEC_ICMP:%.*]] = icmp eq <vscale x 4 x i32> [[MASKED_LOAD]], splat (i32 2)161; CHECK-VF4IC1-NEXT: [[VEC_SEL_TMP:%.*]] = or <vscale x 4 x i1> [[VEC_PHI]], [[VEC_ICMP]]162; CHECK-VF4IC1: [[VEC_SEL:%.*]] = select <vscale x 4 x i1> [[MASK]], <vscale x 4 x i1> [[VEC_SEL_TMP]], <vscale x 4 x i1> [[VEC_PHI]]163; CHECK-VF4IC1: middle.block:164; CHECK-VF4IC1-NEXT: [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[VEC_SEL]])165; CHECK-VF4IC1-NEXT: [[FR:%.*]] = freeze i1 [[OR_RDX]]166; CHECK-VF4IC1-NEXT: {{.*}} = select i1 [[FR]], i32 1, i32 0167; CHECK-VF4IC1: %cmp.n = icmp eq i64 %n, %n.vec168 169; CHECK-VF4IC4-LABEL: @pred_select_const_i32_from_icmp170; CHECK-VF4IC4: vector.body:171entry:172 br label %for.body173 174for.body: ; preds = %entry, %for.inc175 %i.013 = phi i64 [ %inc, %for.inc ], [ 0, %entry ]176 %r.012 = phi i32 [ %r.1, %for.inc ], [ 0, %entry ]177 %arrayidx = getelementptr inbounds i32, ptr %src1, i64 %i.013178 %0 = load i32, ptr %arrayidx, align 4179 %cmp1 = icmp sgt i32 %0, 35180 br i1 %cmp1, label %if.then, label %for.inc181 182if.then: ; preds = %for.body183 %arrayidx2 = getelementptr inbounds i32, ptr %src2, i64 %i.013184 %1 = load i32, ptr %arrayidx2, align 4185 %cmp3 = icmp eq i32 %1, 2186 %spec.select = select i1 %cmp3, i32 1, i32 %r.012187 br label %for.inc188 189for.inc: ; preds = %if.then, %for.body190 %r.1 = phi i32 [ %r.012, %for.body ], [ %spec.select, %if.then ]191 %inc = add nuw nsw i64 %i.013, 1192 %exitcond.not = icmp eq i64 %inc, %n193 br i1 %exitcond.not, label %for.end.loopexit, label %for.body, !llvm.loop !0194 195for.end.loopexit: ; preds = %for.inc196 %r.1.lcssa = phi i32 [ %r.1, %for.inc ]197 ret i32 %r.1.lcssa198}199 200 201attributes #0 = { "target-features"="+sve" }202 203!0 = distinct !{!0, !1}204!1 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}205