174 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt < %s -passes=loop-vectorize -prefer-predicate-over-epilogue=scalar-epilogue -mtriple aarch64-unknown-linux-gnu \3; RUN: -mattr=+sve -force-vector-interleave=1 -force-vector-width=4 -prefer-inloop-reductions -S | FileCheck %s4 5define float @cond_fadd(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %cond, i64 %N){6; CHECK-LABEL: @cond_fadd(7; CHECK-NEXT: entry:8; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()9; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 210; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]11; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]12; CHECK: vector.ph:13; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()14; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 415; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]16; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]17; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]18; CHECK: vector.body:19; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]20; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]21; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[COND:%.*]], i64 [[INDEX]]22; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 423; CHECK-NEXT: [[TMP9:%.*]] = fcmp une <vscale x 4 x float> [[WIDE_LOAD]], splat (float 2.000000e+00)24; CHECK-NEXT: [[TMP10:%.*]] = getelementptr float, ptr [[A:%.*]], i64 [[INDEX]]25; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP10]], <vscale x 4 x i1> [[TMP9]], <vscale x 4 x float> poison)26; CHECK-NEXT: [[TMP12:%.*]] = select fast <vscale x 4 x i1> [[TMP9]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> zeroinitializer27; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP12]])28; CHECK-NEXT: [[TMP14]] = fadd fast float [[VEC_PHI]], [[TMP13]]29; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]30; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]31; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]32; CHECK: middle.block:33; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]34; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]35; CHECK: scalar.ph:36; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]37; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[ENTRY]] ]38; CHECK-NEXT: br label [[FOR_BODY:%.*]]39; CHECK: for.body:40; CHECK-NEXT: [[INDVARS:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_NEXT:%.*]], [[FOR_INC:%.*]] ]41; CHECK-NEXT: [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RES:%.*]], [[FOR_INC]] ]42; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[COND]], i64 [[INDVARS]]43; CHECK-NEXT: [[TMP16:%.*]] = load float, ptr [[ARRAYIDX]], align 444; CHECK-NEXT: [[TOBOOL:%.*]] = fcmp une float [[TMP16]], 2.000000e+0045; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]]46; CHECK: if.then:47; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS]]48; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[ARRAYIDX2]], align 449; CHECK-NEXT: [[FADD:%.*]] = fadd fast float [[RDX]], [[TMP17]]50; CHECK-NEXT: br label [[FOR_INC]]51; CHECK: for.inc:52; CHECK-NEXT: [[RES]] = phi float [ [[FADD]], [[IF_THEN]] ], [ [[RDX]], [[FOR_BODY]] ]53; CHECK-NEXT: [[INDVARS_NEXT]] = add nuw nsw i64 [[INDVARS]], 154; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_NEXT]], [[N]]55; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]56; CHECK: for.end:57; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi float [ [[RES]], [[FOR_INC]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ]58; CHECK-NEXT: ret float [[RES_LCSSA]]59;60entry:61 br label %for.body62 63for.body:64 %indvars = phi i64 [ 0, %entry ], [ %indvars.next, %for.inc ]65 %rdx = phi float [ 1.000000e+00, %entry ], [ %res, %for.inc ]66 %arrayidx = getelementptr inbounds float, ptr %cond, i64 %indvars67 %0 = load float, ptr %arrayidx68 %tobool = fcmp une float %0, 2.000000e+0069 br i1 %tobool, label %if.then, label %for.inc70 71if.then:72 %arrayidx2 = getelementptr inbounds float, ptr %a, i64 %indvars73 %1 = load float, ptr %arrayidx274 %fadd = fadd fast float %rdx, %175 br label %for.inc76 77for.inc:78 %res = phi float [ %fadd, %if.then ], [ %rdx, %for.body ]79 %indvars.next = add nuw nsw i64 %indvars, 180 %exitcond.not = icmp eq i64 %indvars.next, %N81 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !082 83for.end:84 ret float %res85}86 87define float @cond_cmp_sel(ptr noalias %a, ptr noalias %cond, i64 %N) {88; CHECK-LABEL: @cond_cmp_sel(89; CHECK-NEXT: entry:90; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()91; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 292; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]93; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]94; CHECK: vector.ph:95; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()96; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 497; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]98; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]99; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]100; CHECK: vector.body:101; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]102; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[RDX_MINMAX_SELECT:%.*]], [[VECTOR_BODY]] ]103; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[COND:%.*]], i64 [[INDEX]]104; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4105; CHECK-NEXT: [[TMP9:%.*]] = fcmp une <vscale x 4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)106; CHECK-NEXT: [[TMP10:%.*]] = getelementptr float, ptr [[A:%.*]], i64 [[INDEX]]107; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 4 [[TMP10]], <vscale x 4 x i1> [[TMP9]], <vscale x 4 x float> poison)108; CHECK-NEXT: [[TMP12:%.*]] = select fast <vscale x 4 x i1> [[TMP9]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> splat (float 0x47EFFFFFE0000000)109; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> [[TMP12]])110; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp fast olt float [[TMP13]], [[VEC_PHI]]111; CHECK-NEXT: [[RDX_MINMAX_SELECT]] = select fast i1 [[RDX_MINMAX_CMP]], float [[TMP13]], float [[VEC_PHI]]112; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]113; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]114; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]115; CHECK: middle.block:116; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]117; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]118; CHECK: scalar.ph:119; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]120; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[RDX_MINMAX_SELECT]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[ENTRY]] ]121; CHECK-NEXT: br label [[FOR_BODY:%.*]]122; CHECK: for.body:123; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]124; CHECK-NEXT: [[RDX:%.*]] = phi float [ [[RES:%.*]], [[FOR_INC]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]125; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[COND]], i64 [[IV]]126; CHECK-NEXT: [[TMP15:%.*]] = load float, ptr [[ARRAYIDX]], align 4127; CHECK-NEXT: [[TOBOOL:%.*]] = fcmp une float [[TMP15]], 3.000000e+00128; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]]129; CHECK: if.then:130; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]131; CHECK-NEXT: [[TMP16:%.*]] = load float, ptr [[ARRAYIDX2]], align 4132; CHECK-NEXT: [[FCMP:%.*]] = fcmp fast olt float [[RDX]], [[TMP16]]133; CHECK-NEXT: [[FSEL:%.*]] = select fast i1 [[FCMP]], float [[RDX]], float [[TMP16]]134; CHECK-NEXT: br label [[FOR_INC]]135; CHECK: for.inc:136; CHECK-NEXT: [[RES]] = phi float [ [[RDX]], [[FOR_BODY]] ], [ [[FSEL]], [[IF_THEN]] ]137; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1138; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]139; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]140; CHECK: for.end:141; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi float [ [[RES]], [[FOR_INC]] ], [ [[RDX_MINMAX_SELECT]], [[MIDDLE_BLOCK]] ]142; CHECK-NEXT: ret float [[RES_LCSSA]]143;144entry:145 br label %for.body146 147for.body:148 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]149 %rdx = phi float [ %res, %for.inc ], [ 1.000000e+00, %entry ]150 %arrayidx = getelementptr inbounds float, ptr %cond, i64 %iv151 %0 = load float, ptr %arrayidx152 %tobool = fcmp une float %0, 3.000000e+00153 br i1 %tobool, label %if.then, label %for.inc154 155if.then:156 %arrayidx2 = getelementptr inbounds float, ptr %a, i64 %iv157 %1 = load float, ptr %arrayidx2158 %fcmp = fcmp fast olt float %rdx, %1159 %fsel = select fast i1 %fcmp, float %rdx, float %1160 br label %for.inc161 162for.inc:163 %res = phi float [ %rdx, %for.body ], [ %fsel, %if.then ]164 %iv.next = add i64 %iv, 1165 %exitcond.not = icmp eq i64 %iv.next, %N166 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0167 168for.end:169 ret float %res170}171 172!0 = distinct !{!0, !1}173!1 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}174