1820 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 52; RUN: opt -S -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 < %s | FileCheck %s3 4target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"5 6; Float pattern:7; Check vectorization of reduction code which has an fadd instruction after8; an fcmp instruction which compares an array element and 0.9;10; float fcmp_0_fadd_select1(ptr restrict x, const int N) {11; float sum = 0.12; for (int i = 0; i < N; ++i)13; if (x[i] > (float)0.)14; sum += x[i];15; return sum;16; }17 18define float @fcmp_0_fadd_select1(ptr noalias %x, i32 %N) nounwind readonly {19; CHECK-LABEL: define float @fcmp_0_fadd_select1(20; CHECK-SAME: ptr noalias [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {21; CHECK-NEXT: [[ENTRY:.*]]:22; CHECK-NEXT: [[CMP_1:%.*]] = icmp sgt i32 [[N]], 023; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]24; CHECK: [[FOR_HEADER]]:25; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i6426; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[ZEXT]], 427; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]28; CHECK: [[VECTOR_PH]]:29; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[ZEXT]], 430; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[ZEXT]], [[N_MOD_VF]]31; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]32; CHECK: [[VECTOR_BODY]]:33; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]34; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]35; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDEX]]36; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 437; CHECK-NEXT: [[TMP3:%.*]] = fcmp fast ogt <4 x float> [[WIDE_LOAD]], zeroinitializer38; CHECK-NEXT: [[TMP4:%.*]] = fadd fast <4 x float> [[WIDE_LOAD]], [[VEC_PHI]]39; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x float> [[TMP4]], <4 x float> [[VEC_PHI]]40; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 441; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]42; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]43; CHECK: [[MIDDLE_BLOCK]]:44; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP5]])45; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]46; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]47; CHECK: [[SCALAR_PH]]:48; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_HEADER]] ]49; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_HEADER]] ]50; CHECK-NEXT: br label %[[FOR_BODY:.*]]51; CHECK: [[FOR_BODY]]:52; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]53; CHECK-NEXT: [[SUM_1:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]54; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDVARS_IV]]55; CHECK-NEXT: [[TMP8:%.*]] = load float, ptr [[ARRAYIDX]], align 456; CHECK-NEXT: [[CMP_2:%.*]] = fcmp fast ogt float [[TMP8]], 0.000000e+0057; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[TMP8]], [[SUM_1]]58; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], float [[ADD]], float [[SUM_1]]59; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 160; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]61; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]62; CHECK: [[FOR_END_LOOPEXIT]]:63; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi float [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]64; CHECK-NEXT: br label %[[FOR_END]]65; CHECK: [[FOR_END]]:66; CHECK-NEXT: [[TMP9:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]67; CHECK-NEXT: ret float [[TMP9]]68;69entry:70 %cmp.1 = icmp sgt i32 %N, 071 br i1 %cmp.1, label %for.header, label %for.end72 73for.header: ; preds = %entry74 %zext = zext i32 %N to i6475 br label %for.body76 77for.body: ; preds = %header, %for.body78 %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]79 %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]80 %arrayidx = getelementptr inbounds float, ptr %x, i64 %indvars.iv81 %0 = load float, ptr %arrayidx, align 482 %cmp.2 = fcmp fast ogt float %0, 0.000000e+0083 %add = fadd fast float %0, %sum.184 %sum.2 = select i1 %cmp.2, float %add, float %sum.185 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 186 %exitcond = icmp eq i64 %indvars.iv.next, %zext87 br i1 %exitcond, label %for.end, label %for.body88 89for.end: ; preds = %for.body, %entry90 %1 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]91 ret float %192}93 94; Double pattern:95; Check vectorization of reduction code which has an fadd instruction after96; an fcmp instruction which compares an array element and 0.97;98; double fcmp_0_fadd_select2(ptr restrict x, const int N) {99; double sum = 0.100; for (int i = 0; i < N; ++i)101; if (x[i] > 0.)102; sum += x[i];103; return sum;104; }105 106define double @fcmp_0_fadd_select2(ptr noalias %x, i32 %N) nounwind readonly {107; CHECK-LABEL: define double @fcmp_0_fadd_select2(108; CHECK-SAME: ptr noalias [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {109; CHECK-NEXT: [[ENTRY:.*]]:110; CHECK-NEXT: [[CMP_1:%.*]] = icmp sgt i32 [[N]], 0111; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]112; CHECK: [[FOR_HEADER]]:113; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i64114; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[ZEXT]], 4115; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]116; CHECK: [[VECTOR_PH]]:117; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[ZEXT]], 4118; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[ZEXT]], [[N_MOD_VF]]119; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]120; CHECK: [[VECTOR_BODY]]:121; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]122; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]123; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDEX]]124; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP1]], align 4125; CHECK-NEXT: [[TMP3:%.*]] = fcmp fast ogt <4 x double> [[WIDE_LOAD]], zeroinitializer126; CHECK-NEXT: [[TMP4:%.*]] = fadd fast <4 x double> [[WIDE_LOAD]], [[VEC_PHI]]127; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x double> [[TMP4]], <4 x double> [[VEC_PHI]]128; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4129; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]130; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]131; CHECK: [[MIDDLE_BLOCK]]:132; CHECK-NEXT: [[TMP7:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP5]])133; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]134; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]135; CHECK: [[SCALAR_PH]]:136; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_HEADER]] ]137; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_HEADER]] ]138; CHECK-NEXT: br label %[[FOR_BODY:.*]]139; CHECK: [[FOR_BODY]]:140; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]141; CHECK-NEXT: [[SUM_1:%.*]] = phi double [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]142; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV]]143; CHECK-NEXT: [[TMP8:%.*]] = load double, ptr [[ARRAYIDX]], align 4144; CHECK-NEXT: [[CMP_2:%.*]] = fcmp fast ogt double [[TMP8]], 0.000000e+00145; CHECK-NEXT: [[ADD:%.*]] = fadd fast double [[TMP8]], [[SUM_1]]146; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], double [[ADD]], double [[SUM_1]]147; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1148; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]149; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]150; CHECK: [[FOR_END_LOOPEXIT]]:151; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi double [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]152; CHECK-NEXT: br label %[[FOR_END]]153; CHECK: [[FOR_END]]:154; CHECK-NEXT: [[TMP9:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]155; CHECK-NEXT: ret double [[TMP9]]156;157entry:158 %cmp.1 = icmp sgt i32 %N, 0159 br i1 %cmp.1, label %for.header, label %for.end160 161for.header: ; preds = %entry162 %zext = zext i32 %N to i64163 br label %for.body164 165for.body: ; preds = %header, %for.body166 %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]167 %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]168 %arrayidx = getelementptr inbounds double, ptr %x, i64 %indvars.iv169 %0 = load double, ptr %arrayidx, align 4170 %cmp.2 = fcmp fast ogt double %0, 0.000000e+00171 %add = fadd fast double %0, %sum.1172 %sum.2 = select i1 %cmp.2, double %add, double %sum.1173 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1174 %exitcond = icmp eq i64 %indvars.iv.next, %zext175 br i1 %exitcond, label %for.end, label %for.body176 177for.end: ; preds = %for.body, %entry178 %1 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]179 ret double %1180}181 182; Float pattern:183; Check vectorization of reduction code which has an fadd instruction after184; an fcmp instruction which compares an array element and a floating-point185; value.186;187; float fcmp_val_fadd_select1(ptr restrict x, float y, const int N) {188; float sum = 0.189; for (int i = 0; i < N; ++i)190; if (x[i] > y)191; sum += x[i];192; return sum;193; }194 195define float @fcmp_val_fadd_select1(ptr noalias %x, float %y, i32 %N) nounwind readonly {196; CHECK-LABEL: define float @fcmp_val_fadd_select1(197; CHECK-SAME: ptr noalias [[X:%.*]], float [[Y:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {198; CHECK-NEXT: [[ENTRY:.*]]:199; CHECK-NEXT: [[CMP_1:%.*]] = icmp sgt i32 [[N]], 0200; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]201; CHECK: [[FOR_HEADER]]:202; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i64203; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[ZEXT]], 4204; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]205; CHECK: [[VECTOR_PH]]:206; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[ZEXT]], 4207; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[ZEXT]], [[N_MOD_VF]]208; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[Y]], i64 0209; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer210; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]211; CHECK: [[VECTOR_BODY]]:212; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]213; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]214; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDEX]]215; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4216; CHECK-NEXT: [[TMP3:%.*]] = fcmp fast ogt <4 x float> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]217; CHECK-NEXT: [[TMP4:%.*]] = fadd fast <4 x float> [[WIDE_LOAD]], [[VEC_PHI]]218; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x float> [[TMP4]], <4 x float> [[VEC_PHI]]219; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4220; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]221; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]222; CHECK: [[MIDDLE_BLOCK]]:223; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP5]])224; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]225; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]226; CHECK: [[SCALAR_PH]]:227; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_HEADER]] ]228; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_HEADER]] ]229; CHECK-NEXT: br label %[[FOR_BODY:.*]]230; CHECK: [[FOR_BODY]]:231; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]232; CHECK-NEXT: [[SUM_1:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]233; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDVARS_IV]]234; CHECK-NEXT: [[TMP8:%.*]] = load float, ptr [[ARRAYIDX]], align 4235; CHECK-NEXT: [[CMP_2:%.*]] = fcmp fast ogt float [[TMP8]], [[Y]]236; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[TMP8]], [[SUM_1]]237; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], float [[ADD]], float [[SUM_1]]238; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1239; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]240; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]241; CHECK: [[FOR_END_LOOPEXIT]]:242; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi float [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]243; CHECK-NEXT: br label %[[FOR_END]]244; CHECK: [[FOR_END]]:245; CHECK-NEXT: [[TMP9:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]246; CHECK-NEXT: ret float [[TMP9]]247;248entry:249 %cmp.1 = icmp sgt i32 %N, 0250 br i1 %cmp.1, label %for.header, label %for.end251 252for.header: ; preds = %entry253 %zext = zext i32 %N to i64254 br label %for.body255 256for.body: ; preds = %header, %for.body257 %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]258 %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]259 %arrayidx = getelementptr inbounds float, ptr %x, i64 %indvars.iv260 %0 = load float, ptr %arrayidx, align 4261 %cmp.2 = fcmp fast ogt float %0, %y262 %add = fadd fast float %0, %sum.1263 %sum.2 = select i1 %cmp.2, float %add, float %sum.1264 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1265 %exitcond = icmp eq i64 %indvars.iv.next, %zext266 br i1 %exitcond, label %for.end, label %for.body267 268for.end: ; preds = %for.body, %entry269 %1 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]270 ret float %1271}272 273; Double pattern:274; Check vectorization of reduction code which has an fadd instruction after275; an fcmp instruction which compares an array element and a floating-point276; value.277;278; double fcmp_val_fadd_select2(ptr restrict x, double y, const int N) {279; double sum = 0.280; for (int i = 0; i < N; ++i)281; if (x[i] > y)282; sum += x[i];283; return sum;284; }285 286define double @fcmp_val_fadd_select2(ptr noalias %x, double %y, i32 %N) nounwind readonly {287; CHECK-LABEL: define double @fcmp_val_fadd_select2(288; CHECK-SAME: ptr noalias [[X:%.*]], double [[Y:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {289; CHECK-NEXT: [[ENTRY:.*]]:290; CHECK-NEXT: [[CMP_1:%.*]] = icmp sgt i32 [[N]], 0291; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]292; CHECK: [[FOR_HEADER]]:293; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i64294; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[ZEXT]], 4295; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]296; CHECK: [[VECTOR_PH]]:297; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[ZEXT]], 4298; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[ZEXT]], [[N_MOD_VF]]299; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[Y]], i64 0300; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer301; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]302; CHECK: [[VECTOR_BODY]]:303; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]304; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]305; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDEX]]306; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP1]], align 4307; CHECK-NEXT: [[TMP3:%.*]] = fcmp fast ogt <4 x double> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]308; CHECK-NEXT: [[TMP4:%.*]] = fadd fast <4 x double> [[WIDE_LOAD]], [[VEC_PHI]]309; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x double> [[TMP4]], <4 x double> [[VEC_PHI]]310; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4311; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]312; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]313; CHECK: [[MIDDLE_BLOCK]]:314; CHECK-NEXT: [[TMP7:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP5]])315; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]316; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]317; CHECK: [[SCALAR_PH]]:318; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_HEADER]] ]319; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_HEADER]] ]320; CHECK-NEXT: br label %[[FOR_BODY:.*]]321; CHECK: [[FOR_BODY]]:322; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]323; CHECK-NEXT: [[SUM_1:%.*]] = phi double [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]324; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV]]325; CHECK-NEXT: [[TMP8:%.*]] = load double, ptr [[ARRAYIDX]], align 4326; CHECK-NEXT: [[CMP_2:%.*]] = fcmp fast ogt double [[TMP8]], [[Y]]327; CHECK-NEXT: [[ADD:%.*]] = fadd fast double [[TMP8]], [[SUM_1]]328; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], double [[ADD]], double [[SUM_1]]329; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1330; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]331; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]332; CHECK: [[FOR_END_LOOPEXIT]]:333; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi double [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]334; CHECK-NEXT: br label %[[FOR_END]]335; CHECK: [[FOR_END]]:336; CHECK-NEXT: [[TMP9:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]337; CHECK-NEXT: ret double [[TMP9]]338;339entry:340 %cmp.1 = icmp sgt i32 %N, 0341 br i1 %cmp.1, label %for.header, label %for.end342 343for.header: ; preds = %entry344 %zext = zext i32 %N to i64345 br label %for.body346 347for.body: ; preds = %header, %for.body348 %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]349 %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]350 %arrayidx = getelementptr inbounds double, ptr %x, i64 %indvars.iv351 %0 = load double, ptr %arrayidx, align 4352 %cmp.2 = fcmp fast ogt double %0, %y353 %add = fadd fast double %0, %sum.1354 %sum.2 = select i1 %cmp.2, double %add, double %sum.1355 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1356 %exitcond = icmp eq i64 %indvars.iv.next, %zext357 br i1 %exitcond, label %for.end, label %for.body358 359for.end: ; preds = %for.body, %entry360 %1 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]361 ret double %1362}363 364; Float pattern:365; Check vectorization of reduction code which has an fadd instruction after366; an fcmp instruction which compares an array element and another array367; element.368;369; float fcmp_array_elm_fadd_select1(ptr restrict x, ptr restrict y,370; const int N) {371; float sum = 0.372; for (int i = 0; i < N; ++i)373; if (x[i] > y[i])374; sum += x[i];375; return sum;376; }377 378define float @fcmp_array_elm_fadd_select1(ptr noalias %x, ptr noalias %y, i32 %N) nounwind readonly {379; CHECK-LABEL: define float @fcmp_array_elm_fadd_select1(380; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {381; CHECK-NEXT: [[ENTRY:.*]]:382; CHECK-NEXT: [[CMP_1:%.*]] = icmp sgt i32 [[N]], 0383; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]384; CHECK: [[FOR_HEADER]]:385; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i64386; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[ZEXT]], 4387; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]388; CHECK: [[VECTOR_PH]]:389; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[ZEXT]], 4390; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[ZEXT]], [[N_MOD_VF]]391; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]392; CHECK: [[VECTOR_BODY]]:393; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]394; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]395; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDEX]]396; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4397; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 [[INDEX]]398; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP3]], align 4399; CHECK-NEXT: [[TMP5:%.*]] = fcmp fast ogt <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]400; CHECK-NEXT: [[TMP6:%.*]] = fadd fast <4 x float> [[WIDE_LOAD]], [[VEC_PHI]]401; CHECK-NEXT: [[TMP7]] = select <4 x i1> [[TMP5]], <4 x float> [[TMP6]], <4 x float> [[VEC_PHI]]402; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4403; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]404; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]405; CHECK: [[MIDDLE_BLOCK]]:406; CHECK-NEXT: [[TMP9:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP7]])407; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]408; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]409; CHECK: [[SCALAR_PH]]:410; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_HEADER]] ]411; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_HEADER]] ]412; CHECK-NEXT: br label %[[FOR_BODY:.*]]413; CHECK: [[FOR_BODY]]:414; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]415; CHECK-NEXT: [[SUM_1:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]416; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDVARS_IV]]417; CHECK-NEXT: [[TMP10:%.*]] = load float, ptr [[ARRAYIDX_1]], align 4418; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 [[INDVARS_IV]]419; CHECK-NEXT: [[TMP11:%.*]] = load float, ptr [[ARRAYIDX_2]], align 4420; CHECK-NEXT: [[CMP_2:%.*]] = fcmp fast ogt float [[TMP10]], [[TMP11]]421; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[TMP10]], [[SUM_1]]422; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], float [[ADD]], float [[SUM_1]]423; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1424; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]425; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]426; CHECK: [[FOR_END_LOOPEXIT]]:427; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi float [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]428; CHECK-NEXT: br label %[[FOR_END]]429; CHECK: [[FOR_END]]:430; CHECK-NEXT: [[TMP12:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]431; CHECK-NEXT: ret float [[TMP12]]432;433entry:434 %cmp.1 = icmp sgt i32 %N, 0435 br i1 %cmp.1, label %for.header, label %for.end436 437for.header: ; preds = %entry438 %zext = zext i32 %N to i64439 br label %for.body440 441for.body: ; preds = %for.body, %for.header442 %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]443 %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]444 %arrayidx.1 = getelementptr inbounds float, ptr %x, i64 %indvars.iv445 %0 = load float, ptr %arrayidx.1, align 4446 %arrayidx.2 = getelementptr inbounds float, ptr %y, i64 %indvars.iv447 %1 = load float, ptr %arrayidx.2, align 4448 %cmp.2 = fcmp fast ogt float %0, %1449 %add = fadd fast float %0, %sum.1450 %sum.2 = select i1 %cmp.2, float %add, float %sum.1451 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1452 %exitcond = icmp eq i64 %indvars.iv.next, %zext453 br i1 %exitcond, label %for.end, label %for.body454 455for.end: ; preds = %for.body, %entry456 %2 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]457 ret float %2458}459 460; Double pattern:461; Check vectorization of reduction code which has an fadd instruction after462; an fcmp instruction which compares an array element and another array463; element.464;465; double fcmp_array_elm_fadd_select2(ptr restrict x, ptr restrict y,466; const int N) {467; double sum = 0.468; for (int i = 0; i < N; ++i)469; if (x[i] > y[i])470; sum += x[i];471; return sum;472; }473 474define double @fcmp_array_elm_fadd_select2(ptr noalias %x, ptr noalias %y, i32 %N) nounwind readonly {475; CHECK-LABEL: define double @fcmp_array_elm_fadd_select2(476; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {477; CHECK-NEXT: [[ENTRY:.*]]:478; CHECK-NEXT: [[CMP_1:%.*]] = icmp sgt i32 [[N]], 0479; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]480; CHECK: [[FOR_HEADER]]:481; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i64482; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[ZEXT]], 4483; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]484; CHECK: [[VECTOR_PH]]:485; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[ZEXT]], 4486; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[ZEXT]], [[N_MOD_VF]]487; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]488; CHECK: [[VECTOR_BODY]]:489; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]490; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]491; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDEX]]492; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP1]], align 4493; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDEX]]494; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x double>, ptr [[TMP3]], align 4495; CHECK-NEXT: [[TMP5:%.*]] = fcmp fast ogt <4 x double> [[WIDE_LOAD]], [[WIDE_LOAD1]]496; CHECK-NEXT: [[TMP6:%.*]] = fadd fast <4 x double> [[WIDE_LOAD]], [[VEC_PHI]]497; CHECK-NEXT: [[TMP7]] = select <4 x i1> [[TMP5]], <4 x double> [[TMP6]], <4 x double> [[VEC_PHI]]498; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4499; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]500; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]501; CHECK: [[MIDDLE_BLOCK]]:502; CHECK-NEXT: [[TMP9:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP7]])503; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]504; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]505; CHECK: [[SCALAR_PH]]:506; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_HEADER]] ]507; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_HEADER]] ]508; CHECK-NEXT: br label %[[FOR_BODY:.*]]509; CHECK: [[FOR_BODY]]:510; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]511; CHECK-NEXT: [[SUM_1:%.*]] = phi double [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]512; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV]]513; CHECK-NEXT: [[TMP10:%.*]] = load double, ptr [[ARRAYIDX_1]], align 4514; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV]]515; CHECK-NEXT: [[TMP11:%.*]] = load double, ptr [[ARRAYIDX_2]], align 4516; CHECK-NEXT: [[CMP_2:%.*]] = fcmp fast ogt double [[TMP10]], [[TMP11]]517; CHECK-NEXT: [[ADD:%.*]] = fadd fast double [[TMP10]], [[SUM_1]]518; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], double [[ADD]], double [[SUM_1]]519; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1520; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]521; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]522; CHECK: [[FOR_END_LOOPEXIT]]:523; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi double [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]524; CHECK-NEXT: br label %[[FOR_END]]525; CHECK: [[FOR_END]]:526; CHECK-NEXT: [[TMP12:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]527; CHECK-NEXT: ret double [[TMP12]]528;529entry:530 %cmp.1 = icmp sgt i32 %N, 0531 br i1 %cmp.1, label %for.header, label %for.end532 533for.header: ; preds = %entry534 %zext = zext i32 %N to i64535 br label %for.body536 537for.body: ; preds = %for.body, %for.header538 %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]539 %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]540 %arrayidx.1 = getelementptr inbounds double, ptr %x, i64 %indvars.iv541 %0 = load double, ptr %arrayidx.1, align 4542 %arrayidx.2 = getelementptr inbounds double, ptr %y, i64 %indvars.iv543 %1 = load double, ptr %arrayidx.2, align 4544 %cmp.2 = fcmp fast ogt double %0, %1545 %add = fadd fast double %0, %sum.1546 %sum.2 = select i1 %cmp.2, double %add, double %sum.1547 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1548 %exitcond = icmp eq i64 %indvars.iv.next, %zext549 br i1 %exitcond, label %for.end, label %for.body550 551for.end: ; preds = %for.body, %entry552 %2 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]553 ret double %2554}555 556; Float pattern:557; Check vectorization of reduction code which has an fsub instruction after558; an fcmp instruction which compares an array element and 0.559;560; float fcmp_0_fsub_select1(ptr restrict x, const int N) {561; float sum = 0.562; for (int i = 0; i < N; ++i)563; if (x[i] > (float)0.)564; sum -= x[i];565; return sum;566; }567 568define float @fcmp_0_fsub_select1(ptr noalias %x, i32 %N) nounwind readonly {569; CHECK-LABEL: define float @fcmp_0_fsub_select1(570; CHECK-SAME: ptr noalias [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {571; CHECK-NEXT: [[ENTRY:.*]]:572; CHECK-NEXT: [[CMP_1:%.*]] = icmp sgt i32 [[N]], 0573; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]574; CHECK: [[FOR_HEADER]]:575; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i64576; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[ZEXT]], 4577; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]578; CHECK: [[VECTOR_PH]]:579; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[ZEXT]], 4580; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[ZEXT]], [[N_MOD_VF]]581; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]582; CHECK: [[VECTOR_BODY]]:583; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]584; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]585; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDEX]]586; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4587; CHECK-NEXT: [[TMP3:%.*]] = fcmp fast ogt <4 x float> [[WIDE_LOAD]], zeroinitializer588; CHECK-NEXT: [[TMP4:%.*]] = fsub fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]589; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x float> [[TMP4]], <4 x float> [[VEC_PHI]]590; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4591; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]592; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]593; CHECK: [[MIDDLE_BLOCK]]:594; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP5]])595; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]596; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]597; CHECK: [[SCALAR_PH]]:598; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_HEADER]] ]599; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_HEADER]] ]600; CHECK-NEXT: br label %[[FOR_BODY:.*]]601; CHECK: [[FOR_BODY]]:602; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]603; CHECK-NEXT: [[SUM_1:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]604; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDVARS_IV]]605; CHECK-NEXT: [[TMP8:%.*]] = load float, ptr [[ARRAYIDX]], align 4606; CHECK-NEXT: [[CMP_2:%.*]] = fcmp fast ogt float [[TMP8]], 0.000000e+00607; CHECK-NEXT: [[SUB:%.*]] = fsub fast float [[SUM_1]], [[TMP8]]608; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], float [[SUB]], float [[SUM_1]]609; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1610; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]611; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]612; CHECK: [[FOR_END_LOOPEXIT]]:613; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi float [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]614; CHECK-NEXT: br label %[[FOR_END]]615; CHECK: [[FOR_END]]:616; CHECK-NEXT: [[TMP9:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]617; CHECK-NEXT: ret float [[TMP9]]618;619entry:620 %cmp.1 = icmp sgt i32 %N, 0621 br i1 %cmp.1, label %for.header, label %for.end622 623for.header: ; preds = %entry624 %zext = zext i32 %N to i64625 br label %for.body626 627for.body: ; preds = %for.body, %for.header628 %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]629 %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]630 %arrayidx = getelementptr inbounds float, ptr %x, i64 %indvars.iv631 %0 = load float, ptr %arrayidx, align 4632 %cmp.2 = fcmp fast ogt float %0, 0.000000e+00633 %sub = fsub fast float %sum.1, %0634 %sum.2 = select i1 %cmp.2, float %sub, float %sum.1635 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1636 %exitcond = icmp eq i64 %indvars.iv.next, %zext637 br i1 %exitcond, label %for.end, label %for.body638 639for.end: ; preds = %for.body, %entry640 %1 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]641 ret float %1642}643 644; Float pattern:645; Check that is not vectorized if fp-instruction has no fast-math property.646; float fcmp_0_fsub_select1_novectorize(ptr restrict x, const int N) {647; float sum = 0.648; for (int i = 0; i < N; ++i)649; if (x[i] > (float)0.)650; sum -= x[i];651; return sum;652; }653 654define float @fcmp_0_fsub_select1_novectorize(ptr noalias %x, i32 %N) nounwind readonly {655; CHECK-LABEL: define float @fcmp_0_fsub_select1_novectorize(656; CHECK-SAME: ptr noalias [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {657; CHECK-NEXT: [[ENTRY:.*]]:658; CHECK-NEXT: [[CMP_1:%.*]] = icmp sgt i32 [[N]], 0659; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]660; CHECK: [[FOR_HEADER]]:661; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i64662; CHECK-NEXT: br label %[[FOR_BODY:.*]]663; CHECK: [[FOR_BODY]]:664; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[FOR_HEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]665; CHECK-NEXT: [[SUM_1:%.*]] = phi float [ 0.000000e+00, %[[FOR_HEADER]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]666; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDVARS_IV]]667; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4668; CHECK-NEXT: [[CMP_2:%.*]] = fcmp ogt float [[TMP0]], 0.000000e+00669; CHECK-NEXT: [[SUB:%.*]] = fsub float [[SUM_1]], [[TMP0]]670; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], float [[SUB]], float [[SUM_1]]671; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1672; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]673; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT:.*]], label %[[FOR_BODY]]674; CHECK: [[FOR_END_LOOPEXIT]]:675; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi float [ [[SUM_2]], %[[FOR_BODY]] ]676; CHECK-NEXT: br label %[[FOR_END]]677; CHECK: [[FOR_END]]:678; CHECK-NEXT: [[TMP1:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]679; CHECK-NEXT: ret float [[TMP1]]680;681entry:682 %cmp.1 = icmp sgt i32 %N, 0683 br i1 %cmp.1, label %for.header, label %for.end684 685for.header: ; preds = %entry686 %zext = zext i32 %N to i64687 br label %for.body688 689for.body: ; preds = %for.body, %for.header690 %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]691 %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]692 %arrayidx = getelementptr inbounds float, ptr %x, i64 %indvars.iv693 %0 = load float, ptr %arrayidx, align 4694 %cmp.2 = fcmp ogt float %0, 0.000000e+00695 %sub = fsub float %sum.1, %0696 %sum.2 = select i1 %cmp.2, float %sub, float %sum.1697 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1698 %exitcond = icmp eq i64 %indvars.iv.next, %zext699 br i1 %exitcond, label %for.end, label %for.body700 701for.end: ; preds = %for.body, %entry702 %1 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]703 ret float %1704}705 706; Double pattern:707; Check vectorization of reduction code which has an fsub instruction after708; an fcmp instruction which compares an array element and 0.709;710; double fcmp_0_fsub_select2(ptr restrict x, const int N) {711; double sum = 0.712; for (int i = 0; i < N; ++i)713; if (x[i] > 0.)714; sum -= x[i];715; return sum;716; }717 718define double @fcmp_0_fsub_select2(ptr noalias %x, i32 %N) nounwind readonly {719; CHECK-LABEL: define double @fcmp_0_fsub_select2(720; CHECK-SAME: ptr noalias [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {721; CHECK-NEXT: [[ENTRY:.*]]:722; CHECK-NEXT: [[CMP_1:%.*]] = icmp sgt i32 [[N]], 0723; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]724; CHECK: [[FOR_HEADER]]:725; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i64726; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[ZEXT]], 4727; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]728; CHECK: [[VECTOR_PH]]:729; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[ZEXT]], 4730; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[ZEXT]], [[N_MOD_VF]]731; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]732; CHECK: [[VECTOR_BODY]]:733; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]734; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]735; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDEX]]736; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP1]], align 4737; CHECK-NEXT: [[TMP3:%.*]] = fcmp fast ogt <4 x double> [[WIDE_LOAD]], zeroinitializer738; CHECK-NEXT: [[TMP4:%.*]] = fsub fast <4 x double> [[VEC_PHI]], [[WIDE_LOAD]]739; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x double> [[TMP4]], <4 x double> [[VEC_PHI]]740; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4741; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]742; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]743; CHECK: [[MIDDLE_BLOCK]]:744; CHECK-NEXT: [[TMP7:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP5]])745; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]746; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]747; CHECK: [[SCALAR_PH]]:748; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_HEADER]] ]749; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_HEADER]] ]750; CHECK-NEXT: br label %[[FOR_BODY:.*]]751; CHECK: [[FOR_BODY]]:752; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]753; CHECK-NEXT: [[SUM_1:%.*]] = phi double [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]754; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV]]755; CHECK-NEXT: [[TMP8:%.*]] = load double, ptr [[ARRAYIDX]], align 4756; CHECK-NEXT: [[CMP_2:%.*]] = fcmp fast ogt double [[TMP8]], 0.000000e+00757; CHECK-NEXT: [[SUB:%.*]] = fsub fast double [[SUM_1]], [[TMP8]]758; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], double [[SUB]], double [[SUM_1]]759; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1760; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]761; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]762; CHECK: [[FOR_END_LOOPEXIT]]:763; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi double [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]764; CHECK-NEXT: br label %[[FOR_END]]765; CHECK: [[FOR_END]]:766; CHECK-NEXT: [[TMP9:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]767; CHECK-NEXT: ret double [[TMP9]]768;769entry:770 %cmp.1 = icmp sgt i32 %N, 0771 br i1 %cmp.1, label %for.header, label %for.end772 773for.header: ; preds = %entry774 %zext = zext i32 %N to i64775 br label %for.body776 777for.body: ; preds = %for.body, %for.header778 %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]779 %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]780 %arrayidx = getelementptr inbounds double, ptr %x, i64 %indvars.iv781 %0 = load double, ptr %arrayidx, align 4782 %cmp.2 = fcmp fast ogt double %0, 0.000000e+00783 %sub = fsub fast double %sum.1, %0784 %sum.2 = select i1 %cmp.2, double %sub, double %sum.1785 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1786 %exitcond = icmp eq i64 %indvars.iv.next, %zext787 br i1 %exitcond, label %for.end, label %for.body788 789for.end: ; preds = %for.body, %entry790 %1 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]791 ret double %1792}793 794; Double pattern:795; Check that is not vectorized if fp-instruction has no fast-math property.796;797; double fcmp_0_fsub_select2_notvectorize(ptr restrict x, const int N) {798; double sum = 0.799; for (int i = 0; i < N; ++i)800; if (x[i] > 0.)801; sum -= x[i];802; return sum;803; }804 805define double @fcmp_0_fsub_select2_notvectorize(ptr noalias %x, i32 %N) nounwind readonly {806; CHECK-LABEL: define double @fcmp_0_fsub_select2_notvectorize(807; CHECK-SAME: ptr noalias [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {808; CHECK-NEXT: [[ENTRY:.*]]:809; CHECK-NEXT: [[CMP_1:%.*]] = icmp sgt i32 [[N]], 0810; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]811; CHECK: [[FOR_HEADER]]:812; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i64813; CHECK-NEXT: br label %[[FOR_BODY:.*]]814; CHECK: [[FOR_BODY]]:815; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[FOR_HEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]816; CHECK-NEXT: [[SUM_1:%.*]] = phi double [ 0.000000e+00, %[[FOR_HEADER]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]817; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV]]818; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 4819; CHECK-NEXT: [[CMP_2:%.*]] = fcmp ogt double [[TMP0]], 0.000000e+00820; CHECK-NEXT: [[SUB:%.*]] = fsub double [[SUM_1]], [[TMP0]]821; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], double [[SUB]], double [[SUM_1]]822; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1823; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]824; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT:.*]], label %[[FOR_BODY]]825; CHECK: [[FOR_END_LOOPEXIT]]:826; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi double [ [[SUM_2]], %[[FOR_BODY]] ]827; CHECK-NEXT: br label %[[FOR_END]]828; CHECK: [[FOR_END]]:829; CHECK-NEXT: [[TMP1:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]830; CHECK-NEXT: ret double [[TMP1]]831;832entry:833 %cmp.1 = icmp sgt i32 %N, 0834 br i1 %cmp.1, label %for.header, label %for.end835 836for.header: ; preds = %entry837 %zext = zext i32 %N to i64838 br label %for.body839 840for.body: ; preds = %for.body, %for.header841 %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]842 %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]843 %arrayidx = getelementptr inbounds double, ptr %x, i64 %indvars.iv844 %0 = load double, ptr %arrayidx, align 4845 %cmp.2 = fcmp ogt double %0, 0.000000e+00846 %sub = fsub double %sum.1, %0847 %sum.2 = select i1 %cmp.2, double %sub, double %sum.1848 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1849 %exitcond = icmp eq i64 %indvars.iv.next, %zext850 br i1 %exitcond, label %for.end, label %for.body851 852for.end: ; preds = %for.body, %entry853 %1 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]854 ret double %1855}856 857; Float pattern:858; Check vectorization of reduction code which has an fmul instruction after859; an fcmp instruction which compares an array element and 0.860;861; float fcmp_0_fmult_select1(ptr restrict x, const int N) {862; float sum = 0.863; for (int i = 0; i < N; ++i)864; if (x[i] > (float)0.)865; sum *= x[i];866; return sum;867; }868 869define float @fcmp_0_fmult_select1(ptr noalias %x, i32 %N) nounwind readonly {870; CHECK-LABEL: define float @fcmp_0_fmult_select1(871; CHECK-SAME: ptr noalias [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {872; CHECK-NEXT: [[ENTRY:.*]]:873; CHECK-NEXT: [[CMP_1:%.*]] = icmp sgt i32 [[N]], 0874; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]875; CHECK: [[FOR_HEADER]]:876; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i64877; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[ZEXT]], 4878; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]879; CHECK: [[VECTOR_PH]]:880; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[ZEXT]], 4881; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[ZEXT]], [[N_MOD_VF]]882; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]883; CHECK: [[VECTOR_BODY]]:884; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]885; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float 0.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]886; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDEX]]887; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4888; CHECK-NEXT: [[TMP3:%.*]] = fcmp fast ogt <4 x float> [[WIDE_LOAD]], zeroinitializer889; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]890; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x float> [[TMP4]], <4 x float> [[VEC_PHI]]891; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4892; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]893; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]894; CHECK: [[MIDDLE_BLOCK]]:895; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[TMP5]])896; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]897; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]898; CHECK: [[SCALAR_PH]]:899; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_HEADER]] ]900; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_HEADER]] ]901; CHECK-NEXT: br label %[[FOR_BODY:.*]]902; CHECK: [[FOR_BODY]]:903; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]904; CHECK-NEXT: [[SUM_1:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]905; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDVARS_IV]]906; CHECK-NEXT: [[TMP8:%.*]] = load float, ptr [[ARRAYIDX]], align 4907; CHECK-NEXT: [[CMP_2:%.*]] = fcmp fast ogt float [[TMP8]], 0.000000e+00908; CHECK-NEXT: [[MULT:%.*]] = fmul fast float [[SUM_1]], [[TMP8]]909; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], float [[MULT]], float [[SUM_1]]910; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1911; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]912; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]913; CHECK: [[FOR_END_LOOPEXIT]]:914; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi float [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]915; CHECK-NEXT: br label %[[FOR_END]]916; CHECK: [[FOR_END]]:917; CHECK-NEXT: [[TMP9:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]918; CHECK-NEXT: ret float [[TMP9]]919;920entry:921 %cmp.1 = icmp sgt i32 %N, 0922 br i1 %cmp.1, label %for.header, label %for.end923 924for.header: ; preds = %entry925 %zext = zext i32 %N to i64926 br label %for.body927 928for.body: ; preds = %for.body, %for.header929 %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]930 %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]931 %arrayidx = getelementptr inbounds float, ptr %x, i64 %indvars.iv932 %0 = load float, ptr %arrayidx, align 4933 %cmp.2 = fcmp fast ogt float %0, 0.000000e+00934 %mult = fmul fast float %sum.1, %0935 %sum.2 = select i1 %cmp.2, float %mult, float %sum.1936 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1937 %exitcond = icmp eq i64 %indvars.iv.next, %zext938 br i1 %exitcond, label %for.end, label %for.body939 940for.end: ; preds = %for.body, %entry941 %1 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]942 ret float %1943}944 945; Float pattern:946; Check that is not vectorized if fp-instruction has no fast-math property.947;948; float fcmp_0_fmult_select1_notvectorize(ptr restrict x, const int N) {949; float sum = 0.950; for (int i = 0; i < N; ++i)951; if (x[i] > (float)0.)952; sum *= x[i];953; return sum;954; }955 956define float @fcmp_0_fmult_select1_notvectorize(ptr noalias %x, i32 %N) nounwind readonly {957; CHECK-LABEL: define float @fcmp_0_fmult_select1_notvectorize(958; CHECK-SAME: ptr noalias [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {959; CHECK-NEXT: [[ENTRY:.*]]:960; CHECK-NEXT: [[CMP_1:%.*]] = icmp sgt i32 [[N]], 0961; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]962; CHECK: [[FOR_HEADER]]:963; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i64964; CHECK-NEXT: br label %[[FOR_BODY:.*]]965; CHECK: [[FOR_BODY]]:966; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[FOR_HEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]967; CHECK-NEXT: [[SUM_1:%.*]] = phi float [ 0.000000e+00, %[[FOR_HEADER]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]968; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDVARS_IV]]969; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4970; CHECK-NEXT: [[CMP_2:%.*]] = fcmp ogt float [[TMP0]], 0.000000e+00971; CHECK-NEXT: [[MULT:%.*]] = fmul float [[SUM_1]], [[TMP0]]972; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], float [[MULT]], float [[SUM_1]]973; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1974; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]975; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT:.*]], label %[[FOR_BODY]]976; CHECK: [[FOR_END_LOOPEXIT]]:977; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi float [ [[SUM_2]], %[[FOR_BODY]] ]978; CHECK-NEXT: br label %[[FOR_END]]979; CHECK: [[FOR_END]]:980; CHECK-NEXT: [[TMP1:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]981; CHECK-NEXT: ret float [[TMP1]]982;983entry:984 %cmp.1 = icmp sgt i32 %N, 0985 br i1 %cmp.1, label %for.header, label %for.end986 987for.header: ; preds = %entry988 %zext = zext i32 %N to i64989 br label %for.body990 991for.body: ; preds = %for.body, %for.header992 %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]993 %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]994 %arrayidx = getelementptr inbounds float, ptr %x, i64 %indvars.iv995 %0 = load float, ptr %arrayidx, align 4996 %cmp.2 = fcmp ogt float %0, 0.000000e+00997 %mult = fmul float %sum.1, %0998 %sum.2 = select i1 %cmp.2, float %mult, float %sum.1999 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 11000 %exitcond = icmp eq i64 %indvars.iv.next, %zext1001 br i1 %exitcond, label %for.end, label %for.body1002 1003for.end: ; preds = %for.body, %entry1004 %1 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]1005 ret float %11006}1007 1008; Double pattern:1009; Check vectorization of reduction code which has an fmul instruction after1010; an fcmp instruction which compares an array element and 0.1011;1012; double fcmp_0_fmult_select2(ptr restrict x, const int N) {1013; double sum = 0.1014; for (int i = 0; i < N; ++i)1015; if (x[i] > 0.)1016; sum *= x[i];1017; return sum;1018; }1019 1020define double @fcmp_0_fmult_select2(ptr noalias %x, i32 %N) nounwind readonly {1021; CHECK-LABEL: define double @fcmp_0_fmult_select2(1022; CHECK-SAME: ptr noalias [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {1023; CHECK-NEXT: [[ENTRY:.*]]:1024; CHECK-NEXT: [[CMP_1:%.*]] = icmp sgt i32 [[N]], 01025; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]1026; CHECK: [[FOR_HEADER]]:1027; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i641028; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[ZEXT]], 41029; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]1030; CHECK: [[VECTOR_PH]]:1031; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[ZEXT]], 41032; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[ZEXT]], [[N_MOD_VF]]1033; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]1034; CHECK: [[VECTOR_BODY]]:1035; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]1036; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ <double 0.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]1037; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDEX]]1038; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP1]], align 41039; CHECK-NEXT: [[TMP3:%.*]] = fcmp fast ogt <4 x double> [[WIDE_LOAD]], zeroinitializer1040; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <4 x double> [[VEC_PHI]], [[WIDE_LOAD]]1041; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x double> [[TMP4]], <4 x double> [[VEC_PHI]]1042; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 41043; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1044; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]1045; CHECK: [[MIDDLE_BLOCK]]:1046; CHECK-NEXT: [[TMP7:%.*]] = call fast double @llvm.vector.reduce.fmul.v4f64(double 1.000000e+00, <4 x double> [[TMP5]])1047; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]1048; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]1049; CHECK: [[SCALAR_PH]]:1050; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_HEADER]] ]1051; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_HEADER]] ]1052; CHECK-NEXT: br label %[[FOR_BODY:.*]]1053; CHECK: [[FOR_BODY]]:1054; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]1055; CHECK-NEXT: [[SUM_1:%.*]] = phi double [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]1056; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV]]1057; CHECK-NEXT: [[TMP8:%.*]] = load double, ptr [[ARRAYIDX]], align 41058; CHECK-NEXT: [[CMP_2:%.*]] = fcmp fast ogt double [[TMP8]], 0.000000e+001059; CHECK-NEXT: [[MULT:%.*]] = fmul fast double [[SUM_1]], [[TMP8]]1060; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], double [[MULT]], double [[SUM_1]]1061; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 11062; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]1063; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]1064; CHECK: [[FOR_END_LOOPEXIT]]:1065; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi double [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]1066; CHECK-NEXT: br label %[[FOR_END]]1067; CHECK: [[FOR_END]]:1068; CHECK-NEXT: [[TMP9:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]1069; CHECK-NEXT: ret double [[TMP9]]1070;1071entry:1072 %cmp.1 = icmp sgt i32 %N, 01073 br i1 %cmp.1, label %for.header, label %for.end1074 1075for.header: ; preds = %entry1076 %zext = zext i32 %N to i641077 br label %for.body1078 1079for.body: ; preds = %for.body, %for.header1080 %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]1081 %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]1082 %arrayidx = getelementptr inbounds double, ptr %x, i64 %indvars.iv1083 %0 = load double, ptr %arrayidx, align 41084 %cmp.2 = fcmp fast ogt double %0, 0.000000e+001085 %mult = fmul fast double %sum.1, %01086 %sum.2 = select i1 %cmp.2, double %mult, double %sum.11087 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 11088 %exitcond = icmp eq i64 %indvars.iv.next, %zext1089 br i1 %exitcond, label %for.end, label %for.body1090 1091for.end: ; preds = %for.body, %entry1092 %1 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]1093 ret double %11094}1095 1096; Double pattern:1097; Check that is not vectorized if fp-instruction has no fast-math property.1098;1099; double fcmp_0_fmult_select2_notvectorize(ptr restrict x, const int N) {1100; double sum = 0.1101; for (int i = 0; i < N; ++i)1102; if (x[i] > 0.)1103; sum *= x[i];1104; return sum;1105; }1106 1107define double @fcmp_0_fmult_select2_notvectorize(ptr noalias %x, i32 %N) nounwind readonly {1108; CHECK-LABEL: define double @fcmp_0_fmult_select2_notvectorize(1109; CHECK-SAME: ptr noalias [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {1110; CHECK-NEXT: [[ENTRY:.*]]:1111; CHECK-NEXT: [[CMP_1:%.*]] = icmp sgt i32 [[N]], 01112; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]1113; CHECK: [[FOR_HEADER]]:1114; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i641115; CHECK-NEXT: br label %[[FOR_BODY:.*]]1116; CHECK: [[FOR_BODY]]:1117; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[FOR_HEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]1118; CHECK-NEXT: [[SUM_1:%.*]] = phi double [ 0.000000e+00, %[[FOR_HEADER]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]1119; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV]]1120; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 41121; CHECK-NEXT: [[CMP_2:%.*]] = fcmp ogt double [[TMP0]], 0.000000e+001122; CHECK-NEXT: [[MULT:%.*]] = fmul double [[SUM_1]], [[TMP0]]1123; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], double [[MULT]], double [[SUM_1]]1124; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 11125; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]1126; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT:.*]], label %[[FOR_BODY]]1127; CHECK: [[FOR_END_LOOPEXIT]]:1128; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi double [ [[SUM_2]], %[[FOR_BODY]] ]1129; CHECK-NEXT: br label %[[FOR_END]]1130; CHECK: [[FOR_END]]:1131; CHECK-NEXT: [[TMP1:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]1132; CHECK-NEXT: ret double [[TMP1]]1133;1134entry:1135 %cmp.1 = icmp sgt i32 %N, 01136 br i1 %cmp.1, label %for.header, label %for.end1137 1138for.header: ; preds = %entry1139 %zext = zext i32 %N to i641140 br label %for.body1141 1142for.body: ; preds = %for.body, %for.header1143 %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]1144 %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]1145 %arrayidx = getelementptr inbounds double, ptr %x, i64 %indvars.iv1146 %0 = load double, ptr %arrayidx, align 41147 %cmp.2 = fcmp ogt double %0, 0.000000e+001148 %mult = fmul double %sum.1, %01149 %sum.2 = select i1 %cmp.2, double %mult, double %sum.11150 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 11151 %exitcond = icmp eq i64 %indvars.iv.next, %zext1152 br i1 %exitcond, label %for.end, label %for.body1153 1154for.end: ; preds = %for.body, %entry1155 %1 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]1156 ret double %11157}1158 1159; Float multi pattern1160; Check vectorisation of reduction code with a pair of selects to different1161; fadd patterns.1162;1163; float fcmp_multi(ptr a, int n) {1164; float sum=0.0;1165; for (int i=0;i<n;i++) {1166; if (a[i]>1.0)1167; sum+=a[i];1168; else if (a[i]<3.0)1169; sum+=2*a[i];1170; else1171; sum+=3*a[i];1172; }1173; return sum;1174; }1175 1176define float @fcmp_multi(ptr nocapture readonly %a, i32 %n) nounwind readonly {1177; CHECK-LABEL: define float @fcmp_multi(1178; CHECK-SAME: ptr readonly captures(none) [[A:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {1179; CHECK-NEXT: [[ENTRY:.*]]:1180; CHECK-NEXT: [[CMP10:%.*]] = icmp sgt i32 [[N]], 01181; CHECK-NEXT: br i1 [[CMP10]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]1182; CHECK: [[FOR_BODY_PREHEADER]]:1183; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i641184; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 41185; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]1186; CHECK: [[VECTOR_PH]]:1187; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 41188; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]1189; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]1190; CHECK: [[VECTOR_BODY]]:1191; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]1192; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]1193; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]1194; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 41195; CHECK-NEXT: [[TMP4:%.*]] = fcmp ule <4 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)1196; CHECK-NEXT: [[TMP5:%.*]] = fcmp olt <4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)1197; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)1198; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer1199; CHECK-NEXT: [[TMP9:%.*]] = fmul fast <4 x float> [[WIDE_LOAD]], splat (float 2.000000e+00)1200; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP6]], <4 x float> [[TMP9]], <4 x float> [[TMP8]]1201; CHECK-NEXT: [[PREDPHI1:%.*]] = select <4 x i1> [[TMP4]], <4 x float> [[PREDPHI]], <4 x float> [[WIDE_LOAD]]1202; CHECK-NEXT: [[TMP10]] = fadd fast <4 x float> [[PREDPHI1]], [[VEC_PHI]]1203; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 41204; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1205; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]1206; CHECK: [[MIDDLE_BLOCK]]:1207; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP10]])1208; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]1209; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]1210; CHECK: [[SCALAR_PH]]:1211; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]1212; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_BODY_PREHEADER]] ]1213; CHECK-NEXT: br label %[[FOR_BODY:.*]]1214; CHECK: [[FOR_BODY]]:1215; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]1216; CHECK-NEXT: [[SUM_011:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]1217; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]1218; CHECK-NEXT: [[TMP13:%.*]] = load float, ptr [[ARRAYIDX]], align 41219; CHECK-NEXT: [[CMP1:%.*]] = fcmp ogt float [[TMP13]], 1.000000e+001220; CHECK-NEXT: br i1 [[CMP1]], label %[[FOR_INC]], label %[[IF_ELSE:.*]]1221; CHECK: [[IF_ELSE]]:1222; CHECK-NEXT: [[CMP8:%.*]] = fcmp olt float [[TMP13]], 3.000000e+001223; CHECK-NEXT: br i1 [[CMP8]], label %[[IF_THEN10:.*]], label %[[IF_ELSE14:.*]]1224; CHECK: [[IF_THEN10]]:1225; CHECK-NEXT: [[MUL:%.*]] = fmul fast float [[TMP13]], 2.000000e+001226; CHECK-NEXT: br label %[[FOR_INC]]1227; CHECK: [[IF_ELSE14]]:1228; CHECK-NEXT: [[MUL17:%.*]] = fmul fast float [[TMP13]], 3.000000e+001229; CHECK-NEXT: br label %[[FOR_INC]]1230; CHECK: [[FOR_INC]]:1231; CHECK-NEXT: [[DOTPN:%.*]] = phi float [ [[MUL]], %[[IF_THEN10]] ], [ [[MUL17]], %[[IF_ELSE14]] ], [ [[TMP13]], %[[FOR_BODY]] ]1232; CHECK-NEXT: [[SUM_1]] = fadd fast float [[DOTPN]], [[SUM_011]]1233; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 11234; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]1235; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]1236; CHECK: [[FOR_END_LOOPEXIT]]:1237; CHECK-NEXT: [[SUM_1_LCSSA:%.*]] = phi float [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]1238; CHECK-NEXT: br label %[[FOR_END]]1239; CHECK: [[FOR_END]]:1240; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_1_LCSSA]], %[[FOR_END_LOOPEXIT]] ]1241; CHECK-NEXT: ret float [[SUM_0_LCSSA]]1242;1243entry:1244 %cmp10 = icmp sgt i32 %n, 01245 br i1 %cmp10, label %for.body.preheader, label %for.end1246 1247for.body.preheader: ; preds = %entry1248 %wide.trip.count = zext i32 %n to i641249 br label %for.body1250 1251for.body: ; preds = %for.inc, %for.body.preheader1252 %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.inc ]1253 %sum.011 = phi float [ 0.000000e+00, %for.body.preheader ], [ %sum.1, %for.inc ]1254 %arrayidx = getelementptr inbounds float, ptr %a, i64 %indvars.iv1255 %0 = load float, ptr %arrayidx, align 41256 %cmp1 = fcmp ogt float %0, 1.000000e+001257 br i1 %cmp1, label %for.inc, label %if.else1258 1259if.else: ; preds = %for.body1260 %cmp8 = fcmp olt float %0, 3.000000e+001261 br i1 %cmp8, label %if.then10, label %if.else141262 1263if.then10: ; preds = %if.else1264 %mul = fmul fast float %0, 2.000000e+001265 br label %for.inc1266 1267if.else14: ; preds = %if.else1268 %mul17 = fmul fast float %0, 3.000000e+001269 br label %for.inc1270 1271for.inc: ; preds = %for.body, %if.else14, %if.then101272 %.pn = phi float [ %mul, %if.then10 ], [ %mul17, %if.else14 ], [ %0, %for.body ]1273 %sum.1 = fadd fast float %.pn, %sum.0111274 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 11275 %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count1276 br i1 %exitcond, label %for.end, label %for.body1277 1278for.end: ; preds = %for.inc, %entry1279 %sum.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %sum.1, %for.inc ]1280 ret float %sum.0.lcssa1281}1282 1283; Float fadd + fsub patterns1284; Check vectorisation of reduction code with a pair of selects to different1285; instructions { fadd, fsub } but equivalent (change in constant).1286;1287; float fcmp_multi(ptr a, int n) {1288; float sum=0.0;1289; for (int i=0;i<n;i++) {1290; if (a[i]>1.0)1291; sum+=a[i];1292; else if (a[i]<3.0)1293; sum-=a[i];1294; }1295; return sum;1296; }1297 1298define float @fcmp_fadd_fsub(ptr nocapture readonly %a, i32 %n) nounwind readonly {1299; CHECK-LABEL: define float @fcmp_fadd_fsub(1300; CHECK-SAME: ptr readonly captures(none) [[A:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {1301; CHECK-NEXT: [[ENTRY:.*]]:1302; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N]], 01303; CHECK-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]1304; CHECK: [[FOR_BODY_PREHEADER]]:1305; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i641306; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 41307; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]1308; CHECK: [[VECTOR_PH]]:1309; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 41310; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]1311; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]1312; CHECK: [[VECTOR_BODY]]:1313; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]1314; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI1:%.*]], %[[VECTOR_BODY]] ]1315; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]1316; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 41317; CHECK-NEXT: [[TMP4:%.*]] = fcmp ule <4 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)1318; CHECK-NEXT: [[TMP8:%.*]] = fcmp uge <4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)1319; CHECK-NEXT: [[TMP6:%.*]] = fsub fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]1320; CHECK-NEXT: [[TMP7:%.*]] = fadd fast <4 x float> [[WIDE_LOAD]], [[VEC_PHI]]1321; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer1322; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP9]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]1323; CHECK-NEXT: [[PREDPHI1]] = select <4 x i1> [[TMP4]], <4 x float> [[PREDPHI]], <4 x float> [[TMP7]]1324; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 41325; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1326; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]1327; CHECK: [[MIDDLE_BLOCK]]:1328; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[PREDPHI1]])1329; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]1330; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]1331; CHECK: [[SCALAR_PH]]:1332; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]1333; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[FOR_BODY_PREHEADER]] ]1334; CHECK-NEXT: br label %[[FOR_BODY:.*]]1335; CHECK: [[FOR_BODY]]:1336; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]1337; CHECK-NEXT: [[SUM_010:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]1338; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]1339; CHECK-NEXT: [[TMP12:%.*]] = load float, ptr [[ARRAYIDX]], align 41340; CHECK-NEXT: [[CMP1:%.*]] = fcmp ogt float [[TMP12]], 1.000000e+001341; CHECK-NEXT: br i1 [[CMP1]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]1342; CHECK: [[IF_THEN]]:1343; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[TMP12]], [[SUM_010]]1344; CHECK-NEXT: br label %[[FOR_INC]]1345; CHECK: [[IF_ELSE]]:1346; CHECK-NEXT: [[CMP8:%.*]] = fcmp olt float [[TMP12]], 3.000000e+001347; CHECK-NEXT: br i1 [[CMP8]], label %[[IF_THEN10:.*]], label %[[FOR_INC]]1348; CHECK: [[IF_THEN10]]:1349; CHECK-NEXT: [[SUB:%.*]] = fsub fast float [[SUM_010]], [[TMP12]]1350; CHECK-NEXT: br label %[[FOR_INC]]1351; CHECK: [[FOR_INC]]:1352; CHECK-NEXT: [[SUM_1]] = phi float [ [[ADD]], %[[IF_THEN]] ], [ [[SUB]], %[[IF_THEN10]] ], [ [[SUM_010]], %[[IF_ELSE]] ]1353; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 11354; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]1355; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]1356; CHECK: [[FOR_END_LOOPEXIT]]:1357; CHECK-NEXT: [[SUM_1_LCSSA:%.*]] = phi float [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]1358; CHECK-NEXT: br label %[[FOR_END]]1359; CHECK: [[FOR_END]]:1360; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_1_LCSSA]], %[[FOR_END_LOOPEXIT]] ]1361; CHECK-NEXT: ret float [[SUM_0_LCSSA]]1362;1363entry:1364 %cmp9 = icmp sgt i32 %n, 01365 br i1 %cmp9, label %for.body.preheader, label %for.end1366 1367for.body.preheader: ; preds = %entry1368 %wide.trip.count = zext i32 %n to i641369 br label %for.body1370 1371for.body: ; preds = %for.inc, %for.body.preheader1372 %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.inc ]1373 %sum.010 = phi float [ 0.000000e+00, %for.body.preheader ], [ %sum.1, %for.inc ]1374 %arrayidx = getelementptr inbounds float, ptr %a, i64 %indvars.iv1375 %0 = load float, ptr %arrayidx, align 41376 %cmp1 = fcmp ogt float %0, 1.000000e+001377 br i1 %cmp1, label %if.then, label %if.else1378 1379if.then: ; preds = %for.body1380 %add = fadd fast float %0, %sum.0101381 br label %for.inc1382 1383if.else: ; preds = %for.body1384 %cmp8 = fcmp olt float %0, 3.000000e+001385 br i1 %cmp8, label %if.then10, label %for.inc1386 1387if.then10: ; preds = %if.else1388 %sub = fsub fast float %sum.010, %01389 br label %for.inc1390 1391for.inc: ; preds = %if.then, %if.then10, %if.else1392 %sum.1 = phi float [ %add, %if.then ], [ %sub, %if.then10 ], [ %sum.010, %if.else ]1393 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 11394 %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count1395 br i1 %exitcond, label %for.end, label %for.body1396 1397for.end: ; preds = %for.inc, %entry1398 %sum.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %sum.1, %for.inc ]1399 ret float %sum.0.lcssa1400}1401 1402; Float fadd + fmul patterns1403; Check lack of vectorisation of reduction code with a pair of non-compatible1404; instructions { fadd, fmul }.1405;1406; float fcmp_multi(ptr a, int n) {1407; float sum=0.0;1408; for (int i=0;i<n;i++) {1409; if (a[i]>1.0)1410; sum+=a[i];1411; else if (a[i]<3.0)1412; sum*=a[i];1413; }1414; return sum;1415; }1416 1417define float @fcmp_fadd_fmul(ptr nocapture readonly %a, i32 %n) nounwind readonly {1418; CHECK-LABEL: define float @fcmp_fadd_fmul(1419; CHECK-SAME: ptr readonly captures(none) [[A:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {1420; CHECK-NEXT: [[ENTRY:.*]]:1421; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[N]], 01422; CHECK-NEXT: br i1 [[CMP9]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]1423; CHECK: [[FOR_BODY_PREHEADER]]:1424; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i641425; CHECK-NEXT: br label %[[FOR_BODY:.*]]1426; CHECK: [[FOR_BODY]]:1427; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]1428; CHECK-NEXT: [[SUM_010:%.*]] = phi float [ 0.000000e+00, %[[FOR_BODY_PREHEADER]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]1429; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]1430; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 41431; CHECK-NEXT: [[CMP1:%.*]] = fcmp ogt float [[TMP0]], 1.000000e+001432; CHECK-NEXT: br i1 [[CMP1]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]1433; CHECK: [[IF_THEN]]:1434; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[TMP0]], [[SUM_010]]1435; CHECK-NEXT: br label %[[FOR_INC]]1436; CHECK: [[IF_ELSE]]:1437; CHECK-NEXT: [[CMP8:%.*]] = fcmp olt float [[TMP0]], 3.000000e+001438; CHECK-NEXT: br i1 [[CMP8]], label %[[IF_THEN10:.*]], label %[[FOR_INC]]1439; CHECK: [[IF_THEN10]]:1440; CHECK-NEXT: [[MUL:%.*]] = fmul fast float [[TMP0]], [[SUM_010]]1441; CHECK-NEXT: br label %[[FOR_INC]]1442; CHECK: [[FOR_INC]]:1443; CHECK-NEXT: [[SUM_1]] = phi float [ [[ADD]], %[[IF_THEN]] ], [ [[MUL]], %[[IF_THEN10]] ], [ [[SUM_010]], %[[IF_ELSE]] ]1444; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 11445; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]1446; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT:.*]], label %[[FOR_BODY]]1447; CHECK: [[FOR_END_LOOPEXIT]]:1448; CHECK-NEXT: [[SUM_1_LCSSA:%.*]] = phi float [ [[SUM_1]], %[[FOR_INC]] ]1449; CHECK-NEXT: br label %[[FOR_END]]1450; CHECK: [[FOR_END]]:1451; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM_1_LCSSA]], %[[FOR_END_LOOPEXIT]] ]1452; CHECK-NEXT: ret float [[SUM_0_LCSSA]]1453;1454entry:1455 %cmp9 = icmp sgt i32 %n, 01456 br i1 %cmp9, label %for.body.preheader, label %for.end1457 1458for.body.preheader: ; preds = %entry1459 %wide.trip.count = zext i32 %n to i641460 br label %for.body1461 1462for.body: ; preds = %for.inc, %for.body.preheader1463 %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.inc ]1464 %sum.010 = phi float [ 0.000000e+00, %for.body.preheader ], [ %sum.1, %for.inc ]1465 %arrayidx = getelementptr inbounds float, ptr %a, i64 %indvars.iv1466 %0 = load float, ptr %arrayidx, align 41467 %cmp1 = fcmp ogt float %0, 1.000000e+001468 br i1 %cmp1, label %if.then, label %if.else1469 1470if.then: ; preds = %for.body1471 %add = fadd fast float %0, %sum.0101472 br label %for.inc1473 1474if.else: ; preds = %for.body1475 %cmp8 = fcmp olt float %0, 3.000000e+001476 br i1 %cmp8, label %if.then10, label %for.inc1477 1478if.then10: ; preds = %if.else1479 %mul = fmul fast float %0, %sum.0101480 br label %for.inc1481 1482for.inc: ; preds = %if.then, %if.then10, %if.else1483 %sum.1 = phi float [ %add, %if.then ], [ %mul, %if.then10 ], [ %sum.010, %if.else ]1484 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 11485 %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count1486 br i1 %exitcond, label %for.end, label %for.body1487 1488for.end: ; preds = %for.inc, %entry1489 %sum.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %sum.1, %for.inc ]1490 ret float %sum.0.lcssa1491}1492 1493; Float fadd + store patterns1494; Check lack of vectorisation of reduction code with a store back, given it1495; has loop dependency on a[i].1496;1497; float fcmp_store_back(float a[], int LEN) {1498; float sum = 0.0;1499; for (int i = 0; i < LEN; i++) {1500; sum += a[i];1501; a[i] = sum;1502; }1503; return sum;1504; }1505 1506define float @fcmp_store_back(ptr nocapture %a, i32 %LEN) nounwind readonly {1507; CHECK-LABEL: define float @fcmp_store_back(1508; CHECK-SAME: ptr captures(none) [[A:%.*]], i32 [[LEN:%.*]]) #[[ATTR0]] {1509; CHECK-NEXT: [[ENTRY:.*]]:1510; CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i32 [[LEN]], 01511; CHECK-NEXT: br i1 [[CMP7]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]1512; CHECK: [[FOR_BODY_PREHEADER]]:1513; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[LEN]] to i641514; CHECK-NEXT: br label %[[FOR_BODY:.*]]1515; CHECK: [[FOR_BODY]]:1516; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]1517; CHECK-NEXT: [[SUM_08:%.*]] = phi float [ 0.000000e+00, %[[FOR_BODY_PREHEADER]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]1518; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]1519; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 41520; CHECK-NEXT: [[ADD]] = fadd fast float [[TMP0]], [[SUM_08]]1521; CHECK-NEXT: store float [[ADD]], ptr [[ARRAYIDX]], align 41522; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 11523; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]1524; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT:.*]], label %[[FOR_BODY]]1525; CHECK: [[FOR_END_LOOPEXIT]]:1526; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[FOR_BODY]] ]1527; CHECK-NEXT: br label %[[FOR_END]]1528; CHECK: [[FOR_END]]:1529; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[ADD_LCSSA]], %[[FOR_END_LOOPEXIT]] ]1530; CHECK-NEXT: ret float [[SUM_0_LCSSA]]1531;1532entry:1533 %cmp7 = icmp sgt i32 %LEN, 01534 br i1 %cmp7, label %for.body.preheader, label %for.end1535 1536for.body.preheader: ; preds = %entry1537 %wide.trip.count = zext i32 %LEN to i641538 br label %for.body1539 1540for.body: ; preds = %for.body, %for.body.preheader1541 %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]1542 %sum.08 = phi float [ 0.000000e+00, %for.body.preheader ], [ %add, %for.body ]1543 %arrayidx = getelementptr inbounds float, ptr %a, i64 %indvars.iv1544 %0 = load float, ptr %arrayidx, align 41545 %add = fadd fast float %0, %sum.081546 store float %add, ptr %arrayidx, align 41547 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 11548 %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count1549 br i1 %exitcond, label %for.end, label %for.body1550 1551for.end: ; preds = %for.body, %entry1552 %sum.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]1553 ret float %sum.0.lcssa1554}1555 1556define i64 @fcmp_0_add_select2(ptr noalias %x, i64 %N) nounwind readonly {1557; CHECK-LABEL: define i64 @fcmp_0_add_select2(1558; CHECK-SAME: ptr noalias [[X:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {1559; CHECK-NEXT: [[ENTRY:.*]]:1560; CHECK-NEXT: [[CMP_1:%.*]] = icmp sgt i64 [[N]], 01561; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]1562; CHECK: [[FOR_HEADER]]:1563; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 41564; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]1565; CHECK: [[VECTOR_PH]]:1566; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 41567; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]1568; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]1569; CHECK: [[VECTOR_BODY]]:1570; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]1571; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]1572; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDEX]]1573; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 41574; CHECK-NEXT: [[TMP3:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD]], zeroinitializer1575; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[VEC_PHI]], splat (i64 2)1576; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x i64> [[TMP4]], <4 x i64> [[VEC_PHI]]1577; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 41578; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1579; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]1580; CHECK: [[MIDDLE_BLOCK]]:1581; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP5]])1582; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]1583; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]1584; CHECK: [[SCALAR_PH]]:1585; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_HEADER]] ]1586; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_HEADER]] ]1587; CHECK-NEXT: br label %[[FOR_BODY:.*]]1588; CHECK: [[FOR_BODY]]:1589; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]1590; CHECK-NEXT: [[SUM_1:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]1591; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDVARS_IV]]1592; CHECK-NEXT: [[TMP8:%.*]] = load float, ptr [[ARRAYIDX]], align 41593; CHECK-NEXT: [[CMP_2:%.*]] = fcmp ogt float [[TMP8]], 0.000000e+001594; CHECK-NEXT: [[ADD:%.*]] = add nsw i64 [[SUM_1]], 21595; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], i64 [[ADD]], i64 [[SUM_1]]1596; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 11597; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]1598; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]1599; CHECK: [[FOR_END_LOOPEXIT]]:1600; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi i64 [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]1601; CHECK-NEXT: br label %[[FOR_END]]1602; CHECK: [[FOR_END]]:1603; CHECK-NEXT: [[TMP9:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]1604; CHECK-NEXT: ret i64 [[TMP9]]1605;1606entry:1607 %cmp.1 = icmp sgt i64 %N, 01608 br i1 %cmp.1, label %for.header, label %for.end1609 1610for.header: ; preds = %entry1611 br label %for.body1612 1613for.body: ; preds = %header, %for.body1614 %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]1615 %sum.1 = phi i64 [ 0, %for.header ], [ %sum.2, %for.body ]1616 %arrayidx = getelementptr inbounds float, ptr %x, i64 %indvars.iv1617 %0 = load float, ptr %arrayidx, align 41618 %cmp.2 = fcmp ogt float %0, 0.000000e+001619 %add = add nsw i64 %sum.1, 21620 %sum.2 = select i1 %cmp.2, i64 %add, i64 %sum.11621 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 11622 %exitcond = icmp eq i64 %indvars.iv.next, %N1623 br i1 %exitcond, label %for.end, label %for.body1624 1625for.end: ; preds = %for.body, %entry1626 %1 = phi i64 [ 0, %entry ], [ %sum.2, %for.body ]1627 ret i64 %11628}1629 1630; FIXME: %indvars.iv.next is poison on first iteration due to sub nuw 0, 1.1631define i32 @fcmp_0_sub_select1(ptr noalias %x, i32 %N) nounwind readonly {1632; CHECK-LABEL: define i32 @fcmp_0_sub_select1(1633; CHECK-SAME: ptr noalias [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {1634; CHECK-NEXT: [[ENTRY:.*]]:1635; CHECK-NEXT: [[CMP_1:%.*]] = icmp sgt i32 [[N]], 01636; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]1637; CHECK: [[FOR_HEADER]]:1638; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i641639; CHECK-NEXT: [[TMP0:%.*]] = sub i64 0, [[ZEXT]]1640; CHECK-NEXT: br label %[[VECTOR_PH:.*]]1641; CHECK: [[VECTOR_PH]]:1642; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 41643; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]1644; CHECK-NEXT: [[IND_END:%.*]] = sub i64 0, [[N_VEC]]1645; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]1646; CHECK: [[VECTOR_BODY]]:1647; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]1648; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]1649; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 0, [[INDEX]]1650; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[OFFSET_IDX]]1651; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[TMP2]], i64 01652; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP3]], i64 -31653; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP4]], align 41654; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x float> [[WIDE_LOAD]], <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>1655; CHECK-NEXT: [[TMP5:%.*]] = fcmp ogt <4 x float> [[REVERSE]], zeroinitializer1656; CHECK-NEXT: [[TMP6:%.*]] = sub <4 x i32> [[VEC_PHI]], splat (i32 2)1657; CHECK-NEXT: [[TMP7]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP6]], <4 x i32> [[VEC_PHI]]1658; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 41659; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1660; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]1661; CHECK: [[MIDDLE_BLOCK]]:1662; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])1663; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]1664; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH:.*]]1665; CHECK: [[SCALAR_PH]]:1666; CHECK-NEXT: br label %[[FOR_BODY:.*]]1667; CHECK: [[FOR_BODY]]:1668; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[IND_END]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]1669; CHECK-NEXT: [[SUM_1:%.*]] = phi i32 [ [[TMP9]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]1670; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDVARS_IV]]1671; CHECK-NEXT: [[TMP10:%.*]] = load float, ptr [[ARRAYIDX]], align 41672; CHECK-NEXT: [[CMP_2:%.*]] = fcmp ogt float [[TMP10]], 0.000000e+001673; CHECK-NEXT: [[SUB:%.*]] = sub nsw i32 [[SUM_1]], 21674; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], i32 [[SUB]], i32 [[SUM_1]]1675; CHECK-NEXT: [[INDVARS_IV_NEXT]] = sub nuw nsw i64 [[INDVARS_IV]], 11676; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]1677; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]1678; CHECK: [[FOR_END_LOOPEXIT]]:1679; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]1680; CHECK-NEXT: br label %[[FOR_END]]1681; CHECK: [[FOR_END]]:1682; CHECK-NEXT: [[TMP11:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]1683; CHECK-NEXT: ret i32 [[TMP11]]1684;1685entry:1686 %cmp.1 = icmp sgt i32 %N, 01687 br i1 %cmp.1, label %for.header, label %for.end1688 1689for.header: ; preds = %entry1690 %zext = zext i32 %N to i641691 br label %for.body1692 1693for.body: ; preds = %header, %for.body1694 %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]1695 %sum.1 = phi i32 [ 0, %for.header ], [ %sum.2, %for.body ]1696 %arrayidx = getelementptr inbounds float, ptr %x, i64 %indvars.iv1697 %0 = load float, ptr %arrayidx, align 41698 %cmp.2 = fcmp ogt float %0, 0.000000e+001699 %sub = sub nsw i32 %sum.1, 21700 %sum.2 = select i1 %cmp.2, i32 %sub, i32 %sum.11701 %indvars.iv.next = sub nuw nsw i64 %indvars.iv, 11702 %exitcond = icmp eq i64 %indvars.iv.next, %zext1703 br i1 %exitcond, label %for.end, label %for.body1704 1705for.end: ; preds = %for.body, %entry1706 %1 = phi i32 [ 0, %entry ], [ %sum.2, %for.body ]1707 ret i32 %11708}1709 1710define i32 @fcmp_0_mult_select1(ptr noalias %x, i32 %N) nounwind readonly {1711; CHECK-LABEL: define i32 @fcmp_0_mult_select1(1712; CHECK-SAME: ptr noalias [[X:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {1713; CHECK-NEXT: [[ENTRY:.*]]:1714; CHECK-NEXT: [[CMP_1:%.*]] = icmp sgt i32 [[N]], 01715; CHECK-NEXT: br i1 [[CMP_1]], label %[[FOR_HEADER:.*]], label %[[FOR_END:.*]]1716; CHECK: [[FOR_HEADER]]:1717; CHECK-NEXT: [[ZEXT:%.*]] = zext i32 [[N]] to i641718; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[ZEXT]], 41719; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]1720; CHECK: [[VECTOR_PH]]:1721; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[ZEXT]], 41722; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[ZEXT]], [[N_MOD_VF]]1723; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]1724; CHECK: [[VECTOR_BODY]]:1725; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]1726; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 1, i32 1>, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]1727; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDEX]]1728; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 41729; CHECK-NEXT: [[TMP3:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD]], zeroinitializer1730; CHECK-NEXT: [[TMP4:%.*]] = mul <4 x i32> [[VEC_PHI]], splat (i32 2)1731; CHECK-NEXT: [[TMP5]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> [[VEC_PHI]]1732; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 41733; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1734; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]1735; CHECK: [[MIDDLE_BLOCK]]:1736; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP5]])1737; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[ZEXT]], [[N_VEC]]1738; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]1739; CHECK: [[SCALAR_PH]]:1740; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_HEADER]] ]1741; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_HEADER]] ]1742; CHECK-NEXT: br label %[[FOR_BODY:.*]]1743; CHECK: [[FOR_BODY]]:1744; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]1745; CHECK-NEXT: [[SUM_1:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_BODY]] ]1746; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDVARS_IV]]1747; CHECK-NEXT: [[TMP8:%.*]] = load float, ptr [[ARRAYIDX]], align 41748; CHECK-NEXT: [[CMP_2:%.*]] = fcmp ogt float [[TMP8]], 0.000000e+001749; CHECK-NEXT: [[MULT:%.*]] = mul nsw i32 [[SUM_1]], 21750; CHECK-NEXT: [[SUM_2]] = select i1 [[CMP_2]], i32 [[MULT]], i32 [[SUM_1]]1751; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 11752; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]1753; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]1754; CHECK: [[FOR_END_LOOPEXIT]]:1755; CHECK-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]1756; CHECK-NEXT: br label %[[FOR_END]]1757; CHECK: [[FOR_END]]:1758; CHECK-NEXT: [[TMP9:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[SUM_2_LCSSA]], %[[FOR_END_LOOPEXIT]] ]1759; CHECK-NEXT: ret i32 [[TMP9]]1760;1761entry:1762 %cmp.1 = icmp sgt i32 %N, 01763 br i1 %cmp.1, label %for.header, label %for.end1764 1765for.header: ; preds = %entry1766 %zext = zext i32 %N to i641767 br label %for.body1768 1769for.body: ; preds = %for.body, %for.header1770 %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]1771 %sum.1 = phi i32 [ 0, %for.header ], [ %sum.2, %for.body ]1772 %arrayidx = getelementptr inbounds float, ptr %x, i64 %indvars.iv1773 %0 = load float, ptr %arrayidx, align 41774 %cmp.2 = fcmp ogt float %0, 0.000000e+001775 %mult = mul nsw i32 %sum.1, 21776 %sum.2 = select i1 %cmp.2, i32 %mult, i32 %sum.11777 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 11778 %exitcond = icmp eq i64 %indvars.iv.next, %zext1779 br i1 %exitcond, label %for.end, label %for.body1780 1781for.end: ; preds = %for.body, %entry1782 %1 = phi i32 [ 0, %entry ], [ %sum.2, %for.body ]1783 ret i32 %11784}1785 1786;.1787; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}1788; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}1789; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}1790; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}1791; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}1792; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}1793; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}1794; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}1795; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}1796; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}1797; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}1798; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}1799; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}1800; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}1801; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]}1802; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META2]], [[META1]]}1803; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]}1804; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META2]], [[META1]]}1805; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]], [[META2]]}1806; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META2]], [[META1]]}1807; CHECK: [[LOOP20]] = distinct !{[[LOOP20]], [[META1]], [[META2]]}1808; CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META2]], [[META1]]}1809; CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META1]], [[META2]]}1810; CHECK: [[LOOP23]] = distinct !{[[LOOP23]], [[META2]], [[META1]]}1811; CHECK: [[LOOP24]] = distinct !{[[LOOP24]], [[META1]], [[META2]]}1812; CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META2]], [[META1]]}1813; CHECK: [[LOOP26]] = distinct !{[[LOOP26]], [[META1]], [[META2]]}1814; CHECK: [[LOOP27]] = distinct !{[[LOOP27]], [[META2]], [[META1]]}1815; CHECK: [[LOOP28]] = distinct !{[[LOOP28]], [[META1]], [[META2]]}1816; CHECK: [[LOOP29]] = distinct !{[[LOOP29]], [[META2]], [[META1]]}1817; CHECK: [[LOOP30]] = distinct !{[[LOOP30]], [[META1]], [[META2]]}1818; CHECK: [[LOOP31]] = distinct !{[[LOOP31]], [[META2]], [[META1]]}1819;.1820