1423 lines · plain
1; RUN: opt < %s -passes=loop-vectorize -mtriple aarch64-unknown-linux-gnu -force-ordered-reductions=false -hints-allow-reordering=false -S 2>%t | FileCheck %s --check-prefix=CHECK-NOT-VECTORIZED2; RUN: opt < %s -passes=loop-vectorize -mtriple aarch64-unknown-linux-gnu -force-ordered-reductions=false -hints-allow-reordering=true -S 2>%t | FileCheck %s --check-prefix=CHECK-UNORDERED3; RUN: opt < %s -passes=loop-vectorize -mtriple aarch64-unknown-linux-gnu -force-ordered-reductions=true -hints-allow-reordering=false -S 2>%t | FileCheck %s --check-prefix=CHECK-ORDERED4; RUN: opt < %s -passes=loop-vectorize -mtriple aarch64-unknown-linux-gnu -force-ordered-reductions=true -hints-allow-reordering=true -S 2>%t | FileCheck %s --check-prefix=CHECK-UNORDERED5; RUN: opt < %s -passes=loop-vectorize -mtriple aarch64-unknown-linux-gnu -hints-allow-reordering=false -S 2>%t | FileCheck %s --check-prefix=CHECK-ORDERED6 7define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) {8; CHECK-ORDERED-LABEL: @fadd_strict9; CHECK-ORDERED: vector.body:10; CHECK-ORDERED: %[[VEC_PHI:.*]] = phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX:.*]], %vector.body ]11; CHECK-ORDERED: %[[LOAD:.*]] = load <8 x float>, ptr12; CHECK-ORDERED: %[[RDX]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[VEC_PHI]], <8 x float> %[[LOAD]])13; CHECK-ORDERED: for.end14; CHECK-ORDERED: %[[PHI:.*]] = phi float [ %[[SCALAR:.*]], %for.body ], [ %[[RDX]], %middle.block ]15; CHECK-ORDERED: ret float %[[PHI]]16 17; CHECK-UNORDERED-LABEL: @fadd_strict18; CHECK-UNORDERED: vector.body19; CHECK-UNORDERED: %[[VEC_PHI:.*]] = phi <8 x float> [ <float 0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %vector.ph ], [ %[[FADD_VEC:.*]], %vector.body ]20; CHECK-UNORDERED: %[[LOAD_VEC:.*]] = load <8 x float>, ptr21; CHECK-UNORDERED: %[[FADD_VEC]] = fadd <8 x float> %[[LOAD_VEC]], %[[VEC_PHI]]22; CHECK-UNORDERED-NOT: call float @llvm.vector.reduce.fadd23; CHECK-UNORDERED: middle.block24; CHECK-UNORDERED: %[[RDX:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> %[[FADD_VEC]])25; CHECK-UNORDERED: for.body26; CHECK-UNORDERED: %[[LOAD:.*]] = load float, ptr27; CHECK-UNORDERED: %[[FADD:.*]] = fadd float %[[LOAD]], {{.*}}28; CHECK-UNORDERED: for.end29; CHECK-UNORDERED: %[[RES:.*]] = phi float [ %[[FADD]], %for.body ], [ %[[RDX]], %middle.block ]30; CHECK-UNORDERED: ret float %[[RES]]31 32; CHECK-NOT-VECTORIZED-LABEL: @fadd_strict33; CHECK-NOT-VECTORIZED-NOT: vector.body34 35entry:36 br label %for.body37 38for.body:39 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]40 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]41 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv42 %0 = load float, ptr %arrayidx, align 443 %add = fadd float %0, %sum.0744 %iv.next = add nuw nsw i64 %iv, 145 %exitcond.not = icmp eq i64 %iv.next, %n46 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !047 48for.end:49 ret float %add50}51 52; Same as above but where fadd has a fast-math flag.53define float @fadd_strict_fmf(ptr noalias nocapture readonly %a, i64 %n) {54; CHECK-ORDERED-LABEL: @fadd_strict_fmf55; CHECK-ORDERED: vector.body:56; CHECK-ORDERED: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %vector.ph ], [ [[RDX:%.*]], %vector.body ]57; CHECK-ORDERED: [[LOAD_VEC:%.*]] = load <8 x float>, ptr58; CHECK-ORDERED: [[RDX]] = call nnan float @llvm.vector.reduce.fadd.v8f32(float [[VEC_PHI]], <8 x float> [[LOAD_VEC]])59; CHECK-ORDERED: for.end:60; CHECK-ORDERED: [[RES:%.*]] = phi float [ [[SCALAR:%.*]], %for.body ], [ [[RDX]], %middle.block ]61; CHECK-ORDERED: ret float [[RES]]62 63; CHECK-UNORDERED-LABEL: @fadd_strict_fmf64; CHECK-UNORDERED: vector.body:65; CHECK-UNORDERED: [[VEC_PHI:%.*]] = phi <8 x float> [ <float 0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %vector.ph ], [ [[FADD_VEC:%.*]], %vector.body ]66; CHECK-UNORDERED: [[LOAD_VEC:%.*]] = load <8 x float>, ptr67; CHECK-UNORDERED: [[FADD_VEC]] = fadd nnan <8 x float> [[LOAD_VEC]], [[VEC_PHI]]68; CHECK-UNORDERED-NOT: @llvm.vector.reduce.fadd69; CHECK-UNORDERED: middle.block:70; CHECK-UNORDERED: [[RDX:%.*]] = call nnan float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[FADD_VEC]])71; CHECK-UNORDERED: for.body:72; CHECK-UNORDERED: [[LOAD:%.*]] = load float, ptr73; CHECK-UNORDERED: [[FADD:%.*]] = fadd nnan float [[LOAD]], {{.*}}74; CHECK-UNORDERED: for.end:75; CHECK-UNORDERED: [[RES:%.*]] = phi float [ [[FADD]], %for.body ], [ [[RDX]], %middle.block ]76; CHECK-UNORDERED: ret float [[RES]]77 78; CHECK-NOT-VECTORIZED-LABEL: @fadd_strict_fmf79; CHECK-NOT-VECTORIZED-NOT: vector.body80 81entry:82 br label %for.body83 84for.body:85 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]86 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]87 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv88 %0 = load float, ptr %arrayidx, align 489 %add = fadd nnan float %0, %sum.0790 %iv.next = add nuw nsw i64 %iv, 191 %exitcond.not = icmp eq i64 %iv.next, %n92 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !093 94for.end:95 ret float %add96}97 98define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) {99; CHECK-ORDERED-LABEL: @fadd_strict_unroll100; CHECK-ORDERED: vector.body:101; CHECK-ORDERED: %[[VEC_PHI1:.*]] = phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX4:.*]], %vector.body ]102; CHECK-ORDERED-NOT: phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX4]], %vector.body ]103; CHECK-ORDERED: %[[LOAD1:.*]] = load <8 x float>, ptr104; CHECK-ORDERED: %[[LOAD2:.*]] = load <8 x float>, ptr105; CHECK-ORDERED: %[[LOAD3:.*]] = load <8 x float>, ptr106; CHECK-ORDERED: %[[LOAD4:.*]] = load <8 x float>, ptr107; CHECK-ORDERED: %[[RDX1:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[VEC_PHI1]], <8 x float> %[[LOAD1]])108; CHECK-ORDERED: %[[RDX2:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[RDX1]], <8 x float> %[[LOAD2]])109; CHECK-ORDERED: %[[RDX3:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[RDX2]], <8 x float> %[[LOAD3]])110; CHECK-ORDERED: %[[RDX4]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[RDX3]], <8 x float> %[[LOAD4]])111; CHECK-ORDERED: for.end112; CHECK-ORDERED: %[[PHI:.*]] = phi float [ %[[SCALAR:.*]], %for.body ], [ %[[RDX4]], %middle.block ]113; CHECK-ORDERED: ret float %[[PHI]]114 115; CHECK-UNORDERED-LABEL: @fadd_strict_unroll116; CHECK-UNORDERED: vector.body117; CHECK-UNORDERED: %[[VEC_PHI1:.*]] = phi <8 x float> [ <float 0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %vector.ph ], [ %[[VEC_FADD1:.*]], %vector.body ]118; CHECK-UNORDERED: %[[VEC_PHI2:.*]] = phi <8 x float> [ splat (float -0.000000e+00), %vector.ph ], [ %[[VEC_FADD2:.*]], %vector.body ]119; CHECK-UNORDERED: %[[VEC_PHI3:.*]] = phi <8 x float> [ splat (float -0.000000e+00), %vector.ph ], [ %[[VEC_FADD3:.*]], %vector.body ]120; CHECK-UNORDERED: %[[VEC_PHI4:.*]] = phi <8 x float> [ splat (float -0.000000e+00), %vector.ph ], [ %[[VEC_FADD4:.*]], %vector.body ]121; CHECK-UNORDERED: %[[VEC_LOAD1:.*]] = load <8 x float>, ptr122; CHECK-UNORDERED: %[[VEC_LOAD2:.*]] = load <8 x float>, ptr123; CHECK-UNORDERED: %[[VEC_LOAD3:.*]] = load <8 x float>, ptr124; CHECK-UNORDERED: %[[VEC_LOAD4:.*]] = load <8 x float>, ptr125; CHECK-UNORDERED: %[[VEC_FADD1]] = fadd <8 x float> %[[VEC_LOAD1]], %[[VEC_PHI1]]126; CHECK-UNORDERED: %[[VEC_FADD2]] = fadd <8 x float> %[[VEC_LOAD2]], %[[VEC_PHI2]]127; CHECK-UNORDERED: %[[VEC_FADD3]] = fadd <8 x float> %[[VEC_LOAD3]], %[[VEC_PHI3]]128; CHECK-UNORDERED: %[[VEC_FADD4]] = fadd <8 x float> %[[VEC_LOAD4]], %[[VEC_PHI4]]129; CHECK-UNORDERED-NOT: call float @llvm.vector.reduce.fadd130; CHECK-UNORDERED: middle.block131; CHECK-UNORDERED: %[[BIN_RDX1:.*]] = fadd <8 x float> %[[VEC_FADD2]], %[[VEC_FADD1]]132; CHECK-UNORDERED: %[[BIN_RDX2:.*]] = fadd <8 x float> %[[VEC_FADD3]], %[[BIN_RDX1]]133; CHECK-UNORDERED: %[[BIN_RDX3:.*]] = fadd <8 x float> %[[VEC_FADD4]], %[[BIN_RDX2]]134; CHECK-UNORDERED: %[[RDX:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> %[[BIN_RDX3]])135; CHECK-UNORDERED: for.body136; CHECK-UNORDERED: %[[LOAD:.*]] = load float, ptr137; CHECK-UNORDERED: %[[FADD:.*]] = fadd float %[[LOAD]], {{.*}}138; CHECK-UNORDERED: for.end139; CHECK-UNORDERED: %[[RES:.*]] = phi float [ %[[FADD]], %for.body ], [ %[[RDX]], %middle.block ]140; CHECK-UNORDERED: ret float %[[RES]]141 142; CHECK-NOT-VECTORIZED-LABEL: @fadd_strict_unroll143; CHECK-NOT-VECTORIZED-NOT: vector.body144 145entry:146 br label %for.body147 148for.body:149 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]150 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]151 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv152 %0 = load float, ptr %arrayidx, align 4153 %add = fadd float %0, %sum.07154 %iv.next = add nuw nsw i64 %iv, 1155 %exitcond.not = icmp eq i64 %iv.next, %n156 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !1157 158for.end:159 ret float %add160}161 162; An additional test for unrolling where we need the last value of the reduction, i.e:163; float sum = 0, sum2;164; for(int i=0; i<N; ++i) {165; sum += ptr[i];166; *ptr2 = sum + 42;167; }168; return sum;169 170define float @fadd_strict_unroll_last_val(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, i64 %n) {171; CHECK-ORDERED-LABEL: @fadd_strict_unroll_last_val172; CHECK-ORDERED: vector.body173; CHECK-ORDERED: %[[VEC_PHI1:.*]] = phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX4:.*]], %vector.body ]174; CHECK-ORDERED-NOT: phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX4]], %vector.body ]175; CHECK-ORDERED: %[[LOAD1:.*]] = load <8 x float>, ptr176; CHECK-ORDERED: %[[LOAD2:.*]] = load <8 x float>, ptr177; CHECK-ORDERED: %[[LOAD3:.*]] = load <8 x float>, ptr178; CHECK-ORDERED: %[[LOAD4:.*]] = load <8 x float>, ptr179; CHECK-ORDERED: %[[RDX1:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[VEC_PHI1]], <8 x float> %[[LOAD1]])180; CHECK-ORDERED: %[[RDX2:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[RDX1]], <8 x float> %[[LOAD2]])181; CHECK-ORDERED: %[[RDX3:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[RDX2]], <8 x float> %[[LOAD3]])182; CHECK-ORDERED: %[[RDX4]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[RDX3]], <8 x float> %[[LOAD4]])183; CHECK-ORDERED: for.body184; CHECK-ORDERED: %[[SUM_PHI:.*]] = phi float [ %[[FADD:.*]], %for.body ], [ {{.*}}, %scalar.ph ]185; CHECK-ORDERED: %[[LOAD5:.*]] = load float, ptr186; CHECK-ORDERED: %[[FADD]] = fadd float %[[SUM_PHI]], %[[LOAD5]]187; CHECK-ORDERED: for.cond.cleanup188; CHECK-ORDERED: %[[FADD_LCSSA:.*]] = phi float [ %[[FADD]], %for.body ], [ %[[RDX4]], %middle.block ]189; CHECK-ORDERED: %[[FADD_42:.*]] = fadd float %[[FADD_LCSSA]], 4.200000e+01190; CHECK-ORDERED: store float %[[FADD_42]], ptr %b191; CHECK-ORDERED: for.end192; CHECK-ORDERED: %[[SUM_LCSSA:.*]] = phi float [ %[[FADD_LCSSA]], %for.cond.cleanup ], [ 0.000000e+00, %entry ]193; CHECK-ORDERED: ret float %[[SUM_LCSSA]]194 195; CHECK-UNORDERED-LABEL: @fadd_strict_unroll_last_val196; CHECK-UNORDERED: vector.body197; CHECK-UNORDERED: %[[VEC_PHI1:.*]] = phi <8 x float> [ <float 0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %vector.ph ], [ %[[VEC_FADD1:.*]], %vector.body ]198; CHECK-UNORDERED: %[[VEC_PHI2:.*]] = phi <8 x float> [ splat (float -0.000000e+00), %vector.ph ], [ %[[VEC_FADD2:.*]], %vector.body ]199; CHECK-UNORDERED: %[[VEC_PHI3:.*]] = phi <8 x float> [ splat (float -0.000000e+00), %vector.ph ], [ %[[VEC_FADD3:.*]], %vector.body ]200; CHECK-UNORDERED: %[[VEC_PHI4:.*]] = phi <8 x float> [ splat (float -0.000000e+00), %vector.ph ], [ %[[VEC_FADD4:.*]], %vector.body ]201; CHECK-UNORDERED: %[[VEC_LOAD1:.*]] = load <8 x float>, ptr202; CHECK-UNORDERED: %[[VEC_LOAD2:.*]] = load <8 x float>, ptr203; CHECK-UNORDERED: %[[VEC_LOAD3:.*]] = load <8 x float>, ptr204; CHECK-UNORDERED: %[[VEC_LOAD4:.*]] = load <8 x float>, ptr205; CHECK-UNORDERED: %[[VEC_FADD1]] = fadd <8 x float> %[[VEC_PHI1]], %[[VEC_LOAD1]]206; CHECK-UNORDERED: %[[VEC_FADD2]] = fadd <8 x float> %[[VEC_PHI2]], %[[VEC_LOAD2]]207; CHECK-UNORDERED: %[[VEC_FADD3]] = fadd <8 x float> %[[VEC_PHI3]], %[[VEC_LOAD3]]208; CHECK-UNORDERED: %[[VEC_FADD4]] = fadd <8 x float> %[[VEC_PHI4]], %[[VEC_LOAD4]]209; CHECK-UNORDERED-NOT: call float @llvm.vector.reduce.fadd210; CHECK-UNORDERED: middle.block211; CHECK-UNORDERED: %[[BIN_RDX1:.*]] = fadd <8 x float> %[[VEC_FADD2]], %[[VEC_FADD1]]212; CHECK-UNORDERED: %[[BIN_RDX2:.*]] = fadd <8 x float> %[[VEC_FADD3]], %[[BIN_RDX1]]213; CHECK-UNORDERED: %[[BIN_RDX3:.*]] = fadd <8 x float> %[[VEC_FADD4]], %[[BIN_RDX2]]214; CHECK-UNORDERED: %[[RDX:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> %[[BIN_RDX3]])215; CHECK-UNORDERED: for.body216; CHECK-UNORDERED: %[[LOAD:.*]] = load float, ptr217; CHECK-UNORDERED: %[[FADD:.*]] = fadd float {{.*}}, %[[LOAD]]218; CHECK-UNORDERED: for.cond.cleanup219; CHECK-UNORDERED: %[[FADD_LCSSA:.*]] = phi float [ %[[FADD]], %for.body ], [ %[[RDX]], %middle.block ]220; CHECK-UNORDERED: %[[FADD_42:.*]] = fadd float %[[FADD_LCSSA]], 4.200000e+01221; CHECK-UNORDERED: store float %[[FADD_42]], ptr %b222; CHECK-UNORDERED: for.end223; CHECK-UNORDERED: %[[SUM_LCSSA:.*]] = phi float [ %[[FADD_LCSSA]], %for.cond.cleanup ], [ 0.000000e+00, %entry ]224; CHECK-UNORDERED: ret float %[[SUM_LCSSA]]225 226; CHECK-NOT-VECTORIZED-LABEL: @fadd_strict_unroll_last_val227; CHECK-NOT-VECTORIZED-NOT: vector.body228 229entry:230 %cmp = icmp sgt i64 %n, 0231 br i1 %cmp, label %for.body, label %for.end232 233for.body:234 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]235 %sum = phi float [ 0.000000e+00, %entry ], [ %fadd, %for.body ]236 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv237 %0 = load float, ptr %arrayidx, align 4238 %fadd = fadd float %sum, %0239 %iv.next = add nuw nsw i64 %iv, 1240 %exitcond.not = icmp eq i64 %iv.next, %n241 br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !1242 243for.cond.cleanup:244 %fadd.lcssa = phi float [ %fadd, %for.body ]245 %fadd2 = fadd float %fadd.lcssa, 4.200000e+01246 store float %fadd2, ptr %b, align 4247 br label %for.end248 249for.end:250 %sum.lcssa = phi float [ %fadd.lcssa, %for.cond.cleanup ], [ 0.000000e+00, %entry ]251 ret float %sum.lcssa252}253 254define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, i64 %n) {255; CHECK-ORDERED-LABEL: @fadd_strict_interleave256; CHECK-ORDERED: entry257; CHECK-ORDERED: %[[ARRAYIDX:.*]] = getelementptr inbounds float, ptr %a, i64 1258; CHECK-ORDERED: %[[LOAD1:.*]] = load float, ptr %a259; CHECK-ORDERED: %[[LOAD2:.*]] = load float, ptr %[[ARRAYIDX]]260; CHECK-ORDERED: vector.body261; CHECK-ORDERED: %[[VEC_PHI1:.*]] = phi float [ %[[LOAD2]], %vector.ph ], [ %[[RDX2:.*]], %vector.body ]262; CHECK-ORDERED: %[[VEC_PHI2:.*]] = phi float [ %[[LOAD1]], %vector.ph ], [ %[[RDX1:.*]], %vector.body ]263; CHECK-ORDERED: %[[WIDE_LOAD:.*]] = load <8 x float>, ptr264; CHECK-ORDERED: %[[STRIDED1:.*]] = shufflevector <8 x float> %[[WIDE_LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>265; CHECK-ORDERED: %[[STRIDED2:.*]] = shufflevector <8 x float> %[[WIDE_LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>266; CHECK-ORDERED: %[[RDX2]] = call float @llvm.vector.reduce.fadd.v4f32(float %[[VEC_PHI1]], <4 x float> %[[STRIDED2]])267; CHECK-ORDERED: %[[RDX1]] = call float @llvm.vector.reduce.fadd.v4f32(float %[[VEC_PHI2]], <4 x float> %[[STRIDED1]])268; CHECK-ORDERED: for.end269; CHECK-ORDERED: ret void270 271; CHECK-UNORDERED-LABEL: @fadd_strict_interleave272; CHECK-UNORDERED: %[[ARRAYIDX:.*]] = getelementptr inbounds float, ptr %a, i64 1273; CHECK-UNORDERED: %[[LOADA1:.*]] = load float, ptr %a274; CHECK-UNORDERED: %[[LOADA2:.*]] = load float, ptr %[[ARRAYIDX]]275; CHECK-UNORDERED: vector.ph276; CHECK-UNORDERED: %[[INS2:.*]] = insertelement <4 x float> splat (float -0.000000e+00), float %[[LOADA2]], i32 0277; CHECK-UNORDERED: %[[INS1:.*]] = insertelement <4 x float> splat (float -0.000000e+00), float %[[LOADA1]], i32 0278; CHECK-UNORDERED: vector.body279; CHECK-UNORDERED: %[[VEC_PHI2:.*]] = phi <4 x float> [ %[[INS2]], %vector.ph ], [ %[[VEC_FADD2:.*]], %vector.body ]280; CHECK-UNORDERED: %[[VEC_PHI1:.*]] = phi <4 x float> [ %[[INS1]], %vector.ph ], [ %[[VEC_FADD1:.*]], %vector.body ]281; CHECK-UNORDERED: %[[WIDE_LOAD:.*]] = load <8 x float>, ptr282; CHECK-UNORDERED: %[[STRIDED1:.*]] = shufflevector <8 x float> %[[WIDE_LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>283; CHECK-UNORDERED: %[[STRIDED2:.*]] = shufflevector <8 x float> %[[WIDE_LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>284; CHECK-UNORDERED: %[[VEC_FADD1]] = fadd <4 x float> %[[STRIDED1:.*]], %[[VEC_PHI1]]285; CHECK-UNORDERED: %[[VEC_FADD2]] = fadd <4 x float> %[[STRIDED2:.*]], %[[VEC_PHI2]]286; CHECK-UNORDERED-NOT: call float @llvm.vector.reduce.fadd287; CHECK-UNORDERED: middle.block288; CHECK-UNORDERED: %[[RDX2:.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> %[[VEC_FADD2]])289; CHECK-UNORDERED: %[[RDX1:.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> %[[VEC_FADD1]])290; CHECK-UNORDERED: for.body291; CHECK-UNORDERED: %[[LOAD1:.*]] = load float, ptr292; CHECK-UNORDERED: %[[FADD1:.*]] = fadd float %[[LOAD1]], {{.*}}293; CHECK-UNORDERED: %[[LOAD2:.*]] = load float, ptr294; CHECK-UNORDERED: %[[FADD2:.*]] = fadd float %[[LOAD2]], {{.*}}295; CHECK-UNORDERED: for.end296; CHECK-UNORDERED: %[[SUM1:.*]] = phi float [ %[[FADD1]], %for.body ], [ %[[RDX1]], %middle.block ]297; CHECK-UNORDERED: %[[SUM2:.*]] = phi float [ %[[FADD2]], %for.body ], [ %[[RDX2]], %middle.block ]298; CHECK-UNORDERED: store float %[[SUM1]]299; CHECK-UNORDERED: store float %[[SUM2]]300; CHECK-UNORDERED: ret void301 302; CHECK-NOT-VECTORIZED-LABEL: @fadd_strict_interleave303; CHECK-NOT-VECTORIZED-NOT: vector.body304 305entry:306 %arrayidxa = getelementptr inbounds float, ptr %a, i64 1307 %a1 = load float, ptr %a, align 4308 %a2 = load float, ptr %arrayidxa, align 4309 br label %for.body310 311for.body:312 %add.phi1 = phi float [ %a2, %entry ], [ %add2, %for.body ]313 %add.phi2 = phi float [ %a1, %entry ], [ %add1, %for.body ]314 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]315 %arrayidxb1 = getelementptr inbounds float, ptr %b, i64 %iv316 %0 = load float, ptr %arrayidxb1, align 4317 %add1 = fadd float %0, %add.phi2318 %or = or disjoint i64 %iv, 1319 %arrayidxb2 = getelementptr inbounds float, ptr %b, i64 %or320 %1 = load float, ptr %arrayidxb2, align 4321 %add2 = fadd float %1, %add.phi1322 %iv.next = add nuw nsw i64 %iv, 2323 %exitcond.not = icmp eq i64 %iv.next, %n324 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !2325 326for.end:327 store float %add1, ptr %a, align 4328 store float %add2, ptr %arrayidxa, align 4329 ret void330}331 332define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, i64 %n) {333; CHECK-ORDERED-LABEL: @fadd_of_sum334; CHECK-ORDERED: vector.body335; CHECK-ORDERED: %[[VEC_PHI1:.*]] = phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX:.*]], %vector.body ]336; CHECK-ORDERED: %[[LOAD1:.*]] = load <4 x float>, ptr337; CHECK-ORDERED: %[[LOAD2:.*]] = load <4 x float>, ptr338; CHECK-ORDERED: %[[ADD:.*]] = fadd <4 x float> %[[LOAD1]], %[[LOAD2]]339; CHECK-ORDERED: %[[RDX]] = call float @llvm.vector.reduce.fadd.v4f32(float %[[VEC_PHI1]], <4 x float> %[[ADD]])340; CHECK-ORDERED: for.end.loopexit341; CHECK-ORDERED: %[[EXIT_PHI:.*]] = phi float [ %[[SCALAR:.*]], %for.body ], [ %[[RDX]], %middle.block ]342; CHECK-ORDERED: for.end343; CHECK-ORDERED: %[[PHI:.*]] = phi float [ 0.000000e+00, %entry ], [ %[[EXIT_PHI]], %for.end.loopexit ]344; CHECK-ORDERED: ret float %[[PHI]]345 346; CHECK-UNORDERED-LABEL: @fadd_of_sum347; CHECK-UNORDERED: vector.body348; CHECK-UNORDERED: %[[VEC_PHI:.*]] = phi <4 x float> [ <float 0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %vector.ph ], [ %[[VEC_FADD2:.*]], %vector.body ]349; CHECK-UNORDERED: %[[VEC_LOAD1:.*]] = load <4 x float>, ptr350; CHECK-UNORDERED: %[[VEC_LOAD2:.*]] = load <4 x float>, ptr351; CHECK-UNORDERED: %[[VEC_FADD1:.*]] = fadd <4 x float> %[[VEC_LOAD1]], %[[VEC_LOAD2]]352; CHECK-UNORDERED: %[[VEC_FADD2]] = fadd <4 x float> %[[VEC_PHI]], %[[VEC_FADD1]]353; CHECK-UNORDERED-NOT: call float @llvm.vector.reduce.fadd354; CHECK-UNORDERED: middle.block355; CHECK-UNORDERED: %[[RDX:.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> %[[VEC_FADD2]])356; CHECK-UNORDERED: for.body357; CHECK-UNORDERED: %[[LOAD1:.*]] = load float, ptr358; CHECK-UNORDERED: %[[LOAD2:.*]] = load float, ptr359; CHECK-UNORDERED: %[[FADD1:.*]] = fadd float %[[LOAD1]], %[[LOAD2]]360; CHECK-UNORDERED: %[[FADD2:.*]] = fadd float {{.*}}, %[[FADD1]]361; CHECK-UNORDERED: for.end.loopexit362; CHECK-UNORDERED: %[[EXIT:.*]] = phi float [ %[[FADD2]], %for.body ], [ %[[RDX]], %middle.block ]363; CHECK-UNORDERED: for.end364; CHECK-UNORDERED: %[[SUM:.*]] = phi float [ 0.000000e+00, %entry ], [ %[[EXIT]], %for.end.loopexit ]365; CHECK-UNORDERED: ret float %[[SUM]]366 367; CHECK-NOT-VECTORIZED-LABEL: @fadd_of_sum368; CHECK-NOT-VECTORIZED-NOT: vector.body369 370entry:371 %arrayidx = getelementptr inbounds float, ptr %a, i64 1372 %0 = load float, ptr %arrayidx, align 4373 %cmp1 = fcmp ogt float %0, 5.000000e-01374 br i1 %cmp1, label %for.body, label %for.end375 376for.body: ; preds = %for.body377 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]378 %res.014 = phi float [ 0.000000e+00, %entry ], [ %rdx, %for.body ]379 %arrayidx2 = getelementptr inbounds float, ptr %a, i64 %iv380 %1 = load float, ptr %arrayidx2, align 4381 %arrayidx4 = getelementptr inbounds float, ptr %b, i64 %iv382 %2 = load float, ptr %arrayidx4, align 4383 %add = fadd float %1, %2384 %rdx = fadd float %res.014, %add385 %iv.next = add nuw nsw i64 %iv, 1386 %exitcond.not = icmp eq i64 %iv.next, %n387 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !2388 389for.end: ; preds = %for.body, %entry390 %res = phi float [ 0.000000e+00, %entry ], [ %rdx, %for.body ]391 ret float %res392}393 394define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, i64 %n) {395; CHECK-ORDERED-LABEL: @fadd_conditional396; CHECK-ORDERED: vector.body:397; CHECK-ORDERED: %[[PHI:.*]] = phi float [ 1.000000e+00, %vector.ph ], [ %[[RDX:.*]], %pred.load.continue6 ]398; CHECK-ORDERED: %[[LOAD1:.*]] = load <4 x float>, ptr399; CHECK-ORDERED: %[[FCMP1:.*]] = fcmp une <4 x float> %[[LOAD1]], zeroinitializer400; CHECK-ORDERED: %[[EXTRACT:.*]] = extractelement <4 x i1> %[[FCMP1]], i32 0401; CHECK-ORDERED: br i1 %[[EXTRACT]], label %pred.load.if, label %pred.load.continue402; CHECK-ORDERED: pred.load.continue6403; CHECK-ORDERED: %[[PHI1:.*]] = phi <4 x float> [ %[[PHI0:.*]], %pred.load.continue4 ], [ %[[INS_ELT:.*]], %pred.load.if5 ]404; CHECK-ORDERED: %[[PRED:.*]] = select <4 x i1> %[[FCMP1]], <4 x float> %[[PHI1]], <4 x float> splat (float 3.000000e+00)405; CHECK-ORDERED: %[[RDX]] = call float @llvm.vector.reduce.fadd.v4f32(float %[[PHI]], <4 x float> %[[PRED]])406; CHECK-ORDERED: for.body407; CHECK-ORDERED: %[[RES_PHI:.*]] = phi float [ %[[MERGE_RDX:.*]], %scalar.ph ], [ %[[FADD:.*]], %for.inc ]408; CHECK-ORDERED: %[[LOAD2:.*]] = load float, ptr409; CHECK-ORDERED: %[[FCMP2:.*]] = fcmp une float %[[LOAD2]], 0.000000e+00410; CHECK-ORDERED: br i1 %[[FCMP2]], label %if.then, label %for.inc411; CHECK-ORDERED: if.then412; CHECK-ORDERED: %[[LOAD3:.*]] = load float, ptr413; CHECK-ORDERED: br label %for.inc414; CHECK-ORDERED: for.inc415; CHECK-ORDERED: %[[PHI2:.*]] = phi float [ %[[LOAD3]], %if.then ], [ 3.000000e+00, %for.body ]416; CHECK-ORDERED: %[[FADD]] = fadd float %[[RES_PHI]], %[[PHI2]]417; CHECK-ORDERED: for.end418; CHECK-ORDERED: %[[RDX_PHI:.*]] = phi float [ %[[FADD]], %for.inc ], [ %[[RDX]], %middle.block ]419; CHECK-ORDERED: ret float %[[RDX_PHI]]420 421; CHECK-UNORDERED-LABEL: @fadd_conditional422; CHECK-UNORDERED: vector.body423; CHECK-UNORDERED: %[[PHI:.*]] = phi <4 x float> [ <float 1.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %vector.ph ], [ %[[VEC_FADD:.*]], %pred.load.continue6 ]424; CHECK-UNORDERED: %[[LOAD1:.*]] = load <4 x float>, ptr425; CHECK-UNORDERED: %[[FCMP1:.*]] = fcmp une <4 x float> %[[LOAD1]], zeroinitializer426; CHECK-UNORDERED: %[[EXTRACT:.*]] = extractelement <4 x i1> %[[FCMP1]], i32 0427; CHECK-UNORDERED: br i1 %[[EXTRACT]], label %pred.load.if, label %pred.load.continue428; CHECK-UNORDERED: pred.load.continue6429; CHECK-UNORDERED: %[[PRED:.*]] = select <4 x i1> %[[FCMP1]], <4 x float> %[[PRED_PHI:.*]], <4 x float> splat (float 3.000000e+00)430; CHECK-UNORDERED: %[[VEC_FADD]] = fadd <4 x float> %[[PHI]], %[[PRED]]431; CHECK-UNORDERED-NOT: call float @llvm.vector.reduce.fadd432; CHECK-UNORDERED: middle.block433; CHECK-UNORDERED: %[[RDX:.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> %[[VEC_FADD]])434; CHECK-UNORDERED: for.body435; CHECK-UNORDERED: %[[RES_PHI:.*]] = phi float [ %[[MERGE_RDX:.*]], %scalar.ph ], [ %[[FADD:.*]], %for.inc ]436; CHECK-UNORDERED: %[[LOAD2:.*]] = load float, ptr437; CHECK-UNORDERED: %[[FCMP2:.*]] = fcmp une float %[[LOAD2]], 0.000000e+00438; CHECK-UNORDERED: br i1 %[[FCMP2]], label %if.then, label %for.inc439; CHECK-UNORDERED: if.then440; CHECK-UNORDERED: %[[LOAD3:.*]] = load float, ptr441; CHECK-UNORDERED: for.inc442; CHECK-UNORDERED: %[[PHI:.*]] = phi float [ %[[LOAD3]], %if.then ], [ 3.000000e+00, %for.body ]443; CHECK-UNORDERED: %[[FADD]] = fadd float %[[RES_PHI]], %[[PHI]]444; CHECK-UNORDERED: for.end445; CHECK-UNORDERED: %[[RDX_PHI:.*]] = phi float [ %[[FADD]], %for.inc ], [ %[[RDX]], %middle.block ]446; CHECK-UNORDERED: ret float %[[RDX_PHI]]447 448; CHECK-NOT-VECTORIZED-LABEL: @fadd_conditional449; CHECK-NOT-VECTORIZED-NOT: vector.body450 451entry:452 br label %for.body453 454for.body: ; preds = %for.body455 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]456 %res = phi float [ 1.000000e+00, %entry ], [ %fadd, %for.inc ]457 %arrayidx = getelementptr inbounds float, ptr %b, i64 %iv458 %0 = load float, ptr %arrayidx, align 4459 %tobool = fcmp une float %0, 0.000000e+00460 br i1 %tobool, label %if.then, label %for.inc461 462if.then: ; preds = %for.body463 %arrayidx2 = getelementptr inbounds float, ptr %a, i64 %iv464 %1 = load float, ptr %arrayidx2, align 4465 br label %for.inc466 467for.inc:468 %phi = phi float [ %1, %if.then ], [ 3.000000e+00, %for.body ]469 %fadd = fadd float %res, %phi470 %iv.next = add nuw nsw i64 %iv, 1471 %exitcond.not = icmp eq i64 %iv.next, %n472 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !2473 474for.end:475 %rdx = phi float [ %fadd, %for.inc ]476 ret float %rdx477}478 479; Test to check masking correct, using the "llvm.loop.vectorize.predicate.enable" attribute480define float @fadd_predicated(ptr noalias nocapture %a, i64 %n) {481; CHECK-ORDERED-LABEL: @fadd_predicated482; CHECK-ORDERED: vector.ph483; CHECK-ORDERED: %[[TRIP_MINUS_ONE:.*]] = sub i64 %n, 1484; CHECK-ORDERED: %[[BROADCAST_INS:.*]] = insertelement <2 x i64> poison, i64 %[[TRIP_MINUS_ONE]], i64 0485; CHECK-ORDERED: %[[SPLAT:.*]] = shufflevector <2 x i64> %[[BROADCAST_INS]], <2 x i64> poison, <2 x i32> zeroinitializer486; CHECK-ORDERED: vector.body487; CHECK-ORDERED: %[[RDX_PHI:.*]] = phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX:.*]], %pred.load.continue2 ]488; CHECK-ORDERED: pred.load.continue2489; CHECK-ORDERED: %[[PHI:.*]] = phi <2 x float> [ %[[PHI0:.*]], %pred.load.continue ], [ %[[INS_ELT:.*]], %pred.load.if1 ]490; CHECK-ORDERED: %[[MASK:.*]] = select <2 x i1> %0, <2 x float> %[[PHI]], <2 x float> splat (float -0.000000e+00)491; CHECK-ORDERED: %[[RDX]] = call float @llvm.vector.reduce.fadd.v2f32(float %[[RDX_PHI]], <2 x float> %[[MASK]])492; CHECK-ORDERED: for.end:493; CHECK-ORDERED: ret float %[[RDX]]494 495; CHECK-UNORDERED-LABEL: @fadd_predicated496; CHECK-UNORDERED: vector.ph497; CHECK-UNORDERED: %[[TRIP_MINUS_ONE:.*]] = sub i64 %n, 1498; CHECK-UNORDERED: %[[BROADCAST_INS:.*]] = insertelement <2 x i64> poison, i64 %[[TRIP_MINUS_ONE]], i64 0499; CHECK-UNORDERED: %[[SPLAT:.*]] = shufflevector <2 x i64> %[[BROADCAST_INS]], <2 x i64> poison, <2 x i32> zeroinitializer500; CHECK-UNORDERED: vector.body501; CHECK-UNORDERED: %[[RDX_PHI:.*]] = phi <2 x float> [ <float 0.000000e+00, float -0.000000e+00>, %vector.ph ], [ %[[FADD:.*]], %pred.load.continue2 ]502; CHECK-UNORDERED: %[[ICMP:.*]] = icmp ule <2 x i64> %vec.ind, %[[SPLAT]]503; CHECK-UNORDERED: pred.load.continue2504; CHECK-UNORDERED: %[[FADD]] = fadd <2 x float> %[[RDX_PHI]], {{.*}}505; CHECK-UNORDERED: %[[MASK:.*]] = select <2 x i1> %[[ICMP]], <2 x float> %[[FADD]], <2 x float> %[[RDX_PHI]]506; CHECK-UNORDERED-NOT: call float @llvm.vector.reduce.fadd507; CHECK-UNORDERED: middle.block508; CHECK-UNORDERED: %[[RDX:.*]] = call float @llvm.vector.reduce.fadd.v2f32(float -0.000000e+00, <2 x float> %[[MASK]])509; CHECK-UNORDERED: for.end510; CHECK-UNORDERED: ret float %[[RDX]]511 512; CHECK-NOT-VECTORIZED-LABEL: @fadd_predicated513; CHECK-NOT-VECTORIZED-NOT: vector.body514 515entry:516 br label %for.body517 518for.body: ; preds = %entry, %for.body519 %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]520 %sum.02 = phi float [ %l7, %for.body ], [ 0.000000e+00, %entry ]521 %l2 = getelementptr inbounds float, ptr %a, i64 %iv522 %l3 = load float, ptr %l2, align 4523 %l7 = fadd float %sum.02, %l3524 %iv.next = add i64 %iv, 1525 %exitcond = icmp eq i64 %iv.next, %n526 br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !3527 528for.end: ; preds = %for.body529 %sum.0.lcssa = phi float [ %l7, %for.body ]530 ret float %sum.0.lcssa531}532 533; Negative test - loop contains multiple fadds which we cannot safely reorder534define float @fadd_multiple(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) {535; CHECK-ORDERED-LABEL: @fadd_multiple536; CHECK-ORDERED-NOT: vector.body537 538; CHECK-UNORDERED-LABEL: @fadd_multiple539; CHECK-UNORDERED: vector.body540; CHECK-UNORDERED: %[[PHI:.*]] = phi <8 x float> [ splat (float -0.000000e+00), %vector.ph ], [ %[[VEC_FADD2:.*]], %vector.body ]541; CHECK-UNORDERED: %[[VEC_LOAD1:.*]] = load <8 x float>, ptr542; CHECK-UNORDERED: %[[VEC_FADD1:.*]] = fadd <8 x float> %[[PHI]], %[[VEC_LOAD1]]543; CHECK-UNORDERED: %[[VEC_LOAD2:.*]] = load <8 x float>, ptr544; CHECK-UNORDERED: %[[VEC_FADD2]] = fadd <8 x float> %[[VEC_FADD1]], %[[VEC_LOAD2]]545; CHECK-UNORDERED: middle.block546; CHECK-UNORDERED: %[[RDX:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> %[[VEC_FADD2]])547; CHECK-UNORDERED: for.body548; CHECK-UNORDERED: %[[SUM:.*]] = phi float [ %bc.merge.rdx, %scalar.ph ], [ %[[FADD2:.*]], %for.body ]549; CHECK-UNORDERED: %[[LOAD1:.*]] = load float, ptr550; CHECK-UNORDERED: %[[FADD1:.*]] = fadd float %sum, %[[LOAD1]]551; CHECK-UNORDERED: %[[LOAD2:.*]] = load float, ptr552; CHECK-UNORDERED: %[[FADD2]] = fadd float %[[FADD1]], %[[LOAD2]]553; CHECK-UNORDERED: for.end554; CHECK-UNORDERED: %[[RET:.*]] = phi float [ %[[FADD2]], %for.body ], [ %[[RDX]], %middle.block ]555; CHECK-UNORDERED: ret float %[[RET]]556 557; CHECK-NOT-VECTORIZED-LABEL: @fadd_multiple558; CHECK-NOT-VECTORIZED-NOT: vector.body559 560entry:561 br label %for.body562 563for.body: ; preds = %entry, %for.body564 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]565 %sum = phi float [ -0.000000e+00, %entry ], [ %add3, %for.body ]566 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv567 %0 = load float, ptr %arrayidx, align 4568 %add = fadd float %sum, %0569 %arrayidx2 = getelementptr inbounds float, ptr %b, i64 %iv570 %1 = load float, ptr %arrayidx2, align 4571 %add3 = fadd float %add, %1572 %iv.next = add nuw nsw i64 %iv, 1573 %exitcond.not = icmp eq i64 %iv.next, %n574 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0575 576for.end: ; preds = %for.body577 %rdx = phi float [ %add3, %for.body ]578 ret float %rdx579}580 581; Negative test - loop contains two fadds and only one fadd has the fast flag,582; which we cannot safely reorder.583define float @fadd_multiple_one_flag(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) {584; CHECK-ORDERED-LABEL: @fadd_multiple_one_flag585; CHECK-ORDERED-NOT: vector.body586 587; CHECK-UNORDERED-LABEL: @fadd_multiple_one_flag588; CHECK-UNORDERED: vector.body589; CHECK-UNORDERED: %[[PHI:.*]] = phi <8 x float> [ splat (float -0.000000e+00), %vector.ph ], [ %[[VEC_FADD2:.*]], %vector.body ]590; CHECK-UNORDERED: %[[VEC_LOAD1:.*]] = load <8 x float>, ptr591; CHECK-UNORDERED: %[[VEC_FADD1:.*]] = fadd <8 x float> %[[PHI]], %[[VEC_LOAD1]]592; CHECK-UNORDERED: %[[VEC_LOAD2:.*]] = load <8 x float>, ptr593; CHECK-UNORDERED: %[[VEC_FADD2]] = fadd fast <8 x float> %[[VEC_FADD1]], %[[VEC_LOAD2]]594; CHECK-UNORDERED: middle.block595; CHECK-UNORDERED: %[[RDX:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> %[[VEC_FADD2]])596; CHECK-UNORDERED: for.body597; CHECK-UNORDERED: %[[SUM:.*]] = phi float [ %bc.merge.rdx, %scalar.ph ], [ %[[FADD2:.*]], %for.body ]598; CHECK-UNORDERED: %[[LOAD1:.*]] = load float, ptr599; CHECK-UNORDERED: %[[FADD1:.*]] = fadd float %sum, %[[LOAD1]]600; CHECK-UNORDERED: %[[LOAD2:.*]] = load float, ptr601; CHECK-UNORDERED: %[[FADD2]] = fadd fast float %[[FADD1]], %[[LOAD2]]602; CHECK-UNORDERED: for.end603; CHECK-UNORDERED: %[[RET:.*]] = phi float [ %[[FADD2]], %for.body ], [ %[[RDX]], %middle.block ]604; CHECK-UNORDERED: ret float %[[RET]]605 606; CHECK-NOT-VECTORIZED-LABEL: @fadd_multiple_one_flag607; CHECK-NOT-VECTORIZED-NOT: vector.body608 609entry:610 br label %for.body611 612for.body: ; preds = %entry, %for.body613 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]614 %sum = phi float [ -0.000000e+00, %entry ], [ %add3, %for.body ]615 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv616 %0 = load float, ptr %arrayidx, align 4617 %add = fadd float %sum, %0618 %arrayidx2 = getelementptr inbounds float, ptr %b, i64 %iv619 %1 = load float, ptr %arrayidx2, align 4620 %add3 = fadd fast float %add, %1621 %iv.next = add nuw nsw i64 %iv, 1622 %exitcond.not = icmp eq i64 %iv.next, %n623 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0624 625for.end: ; preds = %for.body626 %rdx = phi float [ %add3, %for.body ]627 ret float %rdx628}629 630; Tests with both a floating point reduction & induction, e.g.631;632;float fp_iv_rdx_loop(float *values, float init, float * __restrict__ A, int N) {633; float fp_inc = 2.0;634; float x = init;635; float sum = 0.0;636; for (int i=0; i < N; ++i) {637; A[i] = x;638; x += fp_inc;639; sum += values[i];640; }641; return sum;642;}643;644 645; Strict reduction could be performed in-loop, but ordered FP induction variables are not supported646; Note: This test does not use metadata hints, and as such we should not expect the CHECK-UNORDERED case to vectorize, even647; with the -hints-allow-reordering flag set to true.648define float @induction_and_reduction(ptr nocapture readonly %values, float %init, ptr noalias nocapture %A, i64 %N) {649; CHECK-ORDERED-LABEL: @induction_and_reduction650; CHECK-ORDERED-NOT: vector.body651 652; CHECK-UNORDERED-LABEL: @induction_and_reduction653; CHECK-UNORDERED-NOT: vector.body654 655; CHECK-NOT-VECTORIZED-LABEL: @induction_and_reduction656; CHECK-NOT-VECTORIZED-NOT: vector.body657 658entry:659 br label %for.body660 661for.body:662 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]663 %sum.015 = phi float [ 0.000000e+00, %entry ], [ %add3, %for.body ]664 %x.014 = phi float [ %init, %entry ], [ %add, %for.body ]665 %arrayidx = getelementptr inbounds float, ptr %A, i64 %iv666 store float %x.014, ptr %arrayidx, align 4667 %add = fadd float %x.014, 2.000000e+00668 %arrayidx2 = getelementptr inbounds float, ptr %values, i64 %iv669 %0 = load float, ptr %arrayidx2, align 4670 %add3 = fadd float %sum.015, %0671 %iv.next = add nuw nsw i64 %iv, 1672 %exitcond.not = icmp eq i64 %iv.next, %N673 br i1 %exitcond.not, label %for.end, label %for.body674 675for.end:676 ret float %add3677}678 679; As above, but with the FP induction being unordered (fast) the loop can be vectorized with strict reductions680define float @fast_induction_and_reduction(ptr nocapture readonly %values, float %init, ptr noalias nocapture %A, i64 %N) {681; CHECK-ORDERED-LABEL: @fast_induction_and_reduction682; CHECK-ORDERED: vector.ph683; CHECK-ORDERED: %[[INDUCTION:.*]] = fadd fast <4 x float> {{.*}}, <float 0.000000e+00, float 2.000000e+00, float 4.000000e+00, float 6.000000e+00>684; CHECK-ORDERED: vector.body685; CHECK-ORDERED: %[[RDX_PHI:.*]] = phi float [ 0.000000e+00, %vector.ph ], [ %[[FADD2:.*]], %vector.body ]686; CHECK-ORDERED: %[[IND_PHI:.*]] = phi <4 x float> [ %[[INDUCTION]], %vector.ph ], [ %[[VEC_IND_NEXT:.*]], %vector.body ]687; CHECK-ORDERED: %[[LOAD1:.*]] = load <4 x float>, ptr688; CHECK-ORDERED: %[[FADD1:.*]] = call float @llvm.vector.reduce.fadd.v4f32(float %[[RDX_PHI]], <4 x float> %[[LOAD1]])689; CHECK-ORDERED: %[[VEC_IND_NEXT]] = fadd fast <4 x float> %[[IND_PHI]], splat (float 8.000000e+00)690; CHECK-ORDERED: for.body691; CHECK-ORDERED: %[[RDX_SUM_PHI:.*]] = phi float [ {{.*}}, %scalar.ph ], [ %[[FADD2:.*]], %for.body ]692; CHECK-ORDERED: %[[IND_SUM_PHI:.*]] = phi fast float [ {{.*}}, %scalar.ph ], [ %[[ADD_IND:.*]], %for.body ]693; CHECK-ORDERED: store float %[[IND_SUM_PHI]], ptr694; CHECK-ORDERED: %[[ADD_IND]] = fadd fast float %[[IND_SUM_PHI]], 2.000000e+00695; CHECK-ORDERED: %[[LOAD2:.*]] = load float, ptr696; CHECK-ORDERED: %[[FADD2]] = fadd float %[[RDX_SUM_PHI]], %[[LOAD2]]697; CHECK-ORDERED: for.end698; CHECK-ORDERED: %[[RES_PHI:.*]] = phi float [ %[[FADD2]], %for.body ], [ %[[FADD1]], %middle.block ]699; CHECK-ORDERED: ret float %[[RES_PHI]]700 701; CHECK-UNORDERED-LABEL: @fast_induction_and_reduction702; CHECK-UNORDERED: vector.ph703; CHECK-UNORDERED: %[[INDUCTION:.*]] = fadd fast <4 x float> {{.*}}, <float 0.000000e+00, float 2.000000e+00, float 4.000000e+00, float 6.000000e+00>704; CHECK-UNORDERED: vector.body705; CHECK-UNORDERED: %[[RDX_PHI:.*]] = phi <4 x float> [ <float 0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %vector.ph ], [ %[[VEC_FADD:.*]], %vector.body ]706; CHECK-UNORDERED: %[[IND_PHI:.*]] = phi <4 x float> [ %[[INDUCTION]], %vector.ph ], [ %[[VEC_IND_NEXT:.*]], %vector.body ]707; CHECK-UNORDERED: %[[LOAD1:.*]] = load <4 x float>, ptr708; CHECK-UNORDERED: %[[VEC_FADD]] = fadd <4 x float> %[[RDX_PHI]], %[[LOAD1]]709; CHECK-UNORDERED: %[[VEC_IND_NEXT]] = fadd fast <4 x float> %[[IND_PHI]], splat (float 8.000000e+00)710; CHECK-UNORDERED: middle.block:711; CHECK-UNORDERED: %[[VEC_RDX:.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> %[[VEC_FADD]])712; CHECK-UNORDERED: for.body:713; CHECK-UNORDERED: %[[RDX_SUM_PHI:.*]] = phi float [ {{.*}}, %scalar.ph ], [ %[[FADD:.*]], %for.body ]714; CHECK-UNORDERED: %[[IND_SUM_PHI:.*]] = phi fast float [ {{.*}}, %scalar.ph ], [ %[[ADD_IND:.*]], %for.body ]715; CHECK-UNORDERED: store float %[[IND_SUM_PHI]], ptr716; CHECK-UNORDERED: %[[ADD_IND]] = fadd fast float %[[IND_SUM_PHI]], 2.000000e+00717; CHECK-UNORDERED: %[[LOAD2:.*]] = load float, ptr718; CHECK-UNORDERED: %[[FADD]] = fadd float %[[RDX_SUM_PHI]], %[[LOAD2]]719; CHECK-UNORDERED: for.end720; CHECK-UNORDERED: %[[RES_PHI:.*]] = phi float [ %[[FADD]], %for.body ], [ %[[VEC_RDX]], %middle.block ]721; CHECK-UNORDERED: ret float %[[RES_PHI]]722 723; CHECK-NOT-VECTORIZED-LABEL: @fast_induction_and_reduction724; CHECK-NOT-VECTORIZED-NOT: vector.body725 726entry:727 br label %for.body728 729for.body:730 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]731 %sum.015 = phi float [ 0.000000e+00, %entry ], [ %add3, %for.body ]732 %x.014 = phi fast float [ %init, %entry ], [ %add, %for.body ]733 %arrayidx = getelementptr inbounds float, ptr %A, i64 %iv734 store float %x.014, ptr %arrayidx, align 4735 %add = fadd fast float %x.014, 2.000000e+00736 %arrayidx2 = getelementptr inbounds float, ptr %values, i64 %iv737 %0 = load float, ptr %arrayidx2, align 4738 %add3 = fadd float %sum.015, %0739 %iv.next = add nuw nsw i64 %iv, 1740 %exitcond.not = icmp eq i64 %iv.next, %N741 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !2742 743for.end:744 ret float %add3745}746 747; The FP induction is fast, but here we can't vectorize as only one of the reductions is an FAdd that can be performed in-loop748; Note: This test does not use metadata hints, and as such we should not expect the CHECK-UNORDERED case to vectorize, even749; with the -hints-allow-reordering flag set to true.750define float @fast_induction_unordered_reduction(ptr nocapture readonly %values, float %init, ptr noalias nocapture %A, ptr noalias nocapture %B, i64 %N) {751 752; CHECK-ORDERED-LABEL: @fast_induction_unordered_reduction753; CHECK-ORDERED-NOT: vector.body754 755; CHECK-UNORDERED-LABEL: @fast_induction_unordered_reduction756; CHECK-UNORDERED-NOT: vector.body757 758; CHECK-NOT-VECTORIZED-LABEL: @fast_induction_unordered_reduction759; CHECK-NOT-VECTORIZED-NOT: vector.body760 761entry:762 br label %for.body763 764for.body:765 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]766 %sum2.023 = phi float [ 3.000000e+00, %entry ], [ %mul, %for.body ]767 %sum.022 = phi float [ 0.000000e+00, %entry ], [ %add3, %for.body ]768 %x.021 = phi float [ %init, %entry ], [ %add, %for.body ]769 %arrayidx = getelementptr inbounds float, ptr %A, i64 %iv770 store float %x.021, ptr %arrayidx, align 4771 %add = fadd fast float %x.021, 2.000000e+00772 %arrayidx2 = getelementptr inbounds float, ptr %values, i64 %iv773 %0 = load float, ptr %arrayidx2, align 4774 %add3 = fadd float %sum.022, %0775 %mul = fmul float %sum2.023, %0776 %iv.next = add nuw nsw i64 %iv, 1777 %exitcond.not = icmp eq i64 %iv.next, %N778 br i1 %exitcond.not, label %for.end, label %for.body779 780for.end:781 %add6 = fadd float %add3, %mul782 ret float %add6783}784 785; Test reductions for a VF of 1 and a UF > 1.786define float @fadd_scalar_vf(ptr noalias nocapture readonly %a, i64 %n) {787; CHECK-ORDERED-LABEL: @fadd_scalar_vf788; CHECK-ORDERED: vector.body789; CHECK-ORDERED: %[[VEC_PHI:.*]] = phi float [ 0.000000e+00, {{.*}} ], [ %[[FADD4:.*]], %vector.body ]790; CHECK-ORDERED: %[[LOAD1:.*]] = load float, ptr791; CHECK-ORDERED: %[[LOAD2:.*]] = load float, ptr792; CHECK-ORDERED: %[[LOAD3:.*]] = load float, ptr793; CHECK-ORDERED: %[[LOAD4:.*]] = load float, ptr794; CHECK-ORDERED: %[[FADD1:.*]] = fadd float %[[VEC_PHI]], %[[LOAD1]]795; CHECK-ORDERED: %[[FADD2:.*]] = fadd float %[[FADD1]], %[[LOAD2]]796; CHECK-ORDERED: %[[FADD3:.*]] = fadd float %[[FADD2]], %[[LOAD3]]797; CHECK-ORDERED: %[[FADD4]] = fadd float %[[FADD3]], %[[LOAD4]]798; CHECK-ORDERED-NOT: call float @llvm.vector.reduce.fadd799; CHECK-ORDERED: scalar.ph800; CHECK-ORDERED: %[[MERGE_RDX:.*]] = phi float [ %[[FADD4]], %middle.block ], [ 0.000000e+00, %entry ]801; CHECK-ORDERED: for.body802; CHECK-ORDERED: %[[SUM_PHI:.*]] = phi float [ %[[MERGE_RDX]], %scalar.ph ], [ %[[FADD5:.*]], %for.body ]803; CHECK-ORDERED: %[[LOAD5:.*]] = load float, ptr804; CHECK-ORDERED: %[[FADD5]] = fadd float %[[LOAD5]], %[[SUM_PHI]]805; CHECK-ORDERED: for.end806; CHECK-ORDERED: %[[RES_PHI:.*]] = phi float [ %[[FADD5]], %for.body ], [ %[[FADD4]], %middle.block ]807; CHECK-ORDERED: ret float %[[RES_PHI]]808 809; CHECK-UNORDERED-LABEL: @fadd_scalar_vf810; CHECK-UNORDERED: vector.body811; CHECK-UNORDERED: %[[VEC_PHI1:.*]] = phi float [ 0.000000e+00, %vector.ph ], [ %[[FADD1:.*]], %vector.body ]812; CHECK-UNORDERED: %[[VEC_PHI2:.*]] = phi float [ -0.000000e+00, %vector.ph ], [ %[[FADD2:.*]], %vector.body ]813; CHECK-UNORDERED: %[[VEC_PHI3:.*]] = phi float [ -0.000000e+00, %vector.ph ], [ %[[FADD3:.*]], %vector.body ]814; CHECK-UNORDERED: %[[VEC_PHI4:.*]] = phi float [ -0.000000e+00, %vector.ph ], [ %[[FADD4:.*]], %vector.body ]815; CHECK-UNORDERED: %[[LOAD1:.*]] = load float, ptr816; CHECK-UNORDERED: %[[LOAD2:.*]] = load float, ptr817; CHECK-UNORDERED: %[[LOAD3:.*]] = load float, ptr818; CHECK-UNORDERED: %[[LOAD4:.*]] = load float, ptr819; CHECK-UNORDERED: %[[FADD1]] = fadd float %[[LOAD1]], %[[VEC_PHI1]]820; CHECK-UNORDERED: %[[FADD2]] = fadd float %[[LOAD2]], %[[VEC_PHI2]]821; CHECK-UNORDERED: %[[FADD3]] = fadd float %[[LOAD3]], %[[VEC_PHI3]]822; CHECK-UNORDERED: %[[FADD4]] = fadd float %[[LOAD4]], %[[VEC_PHI4]]823; CHECK-UNORDERED-NOT: call float @llvm.vector.reduce.fadd824; CHECK-UNORDERED: middle.block825; CHECK-UNORDERED: %[[BIN_RDX1:.*]] = fadd float %[[FADD2]], %[[FADD1]]826; CHECK-UNORDERED: %[[BIN_RDX2:.*]] = fadd float %[[FADD3]], %[[BIN_RDX1]]827; CHECK-UNORDERED: %[[BIN_RDX3:.*]] = fadd float %[[FADD4]], %[[BIN_RDX2]]828; CHECK-UNORDERED: scalar.ph829; CHECK-UNORDERED: %[[MERGE_RDX:.*]] = phi float [ %[[BIN_RDX3]], %middle.block ], [ 0.000000e+00, %entry ]830; CHECK-UNORDERED: for.body831; CHECK-UNORDERED: %[[SUM_PHI:.*]] = phi float [ %[[MERGE_RDX]], %scalar.ph ], [ %[[FADD5:.*]], %for.body ]832; CHECK-UNORDERED: %[[LOAD5:.*]] = load float, ptr833; CHECK-UNORDERED: %[[FADD5]] = fadd float %[[LOAD5]], %[[SUM_PHI]]834; CHECK-UNORDERED: for.end835; CHECK-UNORDERED: %[[RES_PHI:.*]] = phi float [ %[[FADD5]], %for.body ], [ %[[BIN_RDX3]], %middle.block ]836; CHECK-UNORDERED: ret float %[[RES_PHI]]837 838; CHECK-NOT-VECTORIZED-LABEL: @fadd_scalar_vf839; CHECK-NOT-VECTORIZED-NOT: @vector.body840 841entry:842 br label %for.body843 844for.body:845 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]846 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]847 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv848 %0 = load float, ptr %arrayidx, align 4849 %add = fadd float %0, %sum.07850 %iv.next = add nuw nsw i64 %iv, 1851 %exitcond.not = icmp eq i64 %iv.next, %n852 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !4853 854for.end:855 ret float %add856}857 858; Same as above but where fadd has a fast-math flag.859define float @fadd_scalar_vf_fmf(ptr noalias nocapture readonly %a, i64 %n) {860; CHECK-ORDERED-LABEL: @fadd_scalar_vf_fmf861; CHECK-ORDERED: vector.body:862; CHECK-ORDERED: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %vector.ph ], [ [[FADD4:%.*]], %vector.body ]863; CHECK-ORDERED: [[LOAD1:%.*]] = load float, ptr864; CHECK-ORDERED: [[LOAD2:%.*]] = load float, ptr865; CHECK-ORDERED: [[LOAD3:%.*]] = load float, ptr866; CHECK-ORDERED: [[LOAD4:%.*]] = load float, ptr867; CHECK-ORDERED: [[FADD1:%.*]] = fadd nnan float [[VEC_PHI]], [[LOAD1]]868; CHECK-ORDERED: [[FADD2:%.*]] = fadd nnan float [[FADD1]], [[LOAD2]]869; CHECK-ORDERED: [[FADD3:%.*]] = fadd nnan float [[FADD2]], [[LOAD3]]870; CHECK-ORDERED: [[FADD4]] = fadd nnan float [[FADD3]], [[LOAD4]]871; CHECK-ORDERED-NOT: @llvm.vector.reduce.fadd872; CHECK-ORDERED: scalar.ph:873; CHECK-ORDERED: [[MERGE_RDX:%.*]] = phi float [ [[FADD4]], %middle.block ], [ 0.000000e+00, %entry ]874; CHECK-ORDERED: for.body:875; CHECK-ORDERED: [[SUM_07:%.*]] = phi float [ [[MERGE_RDX]], %scalar.ph ], [ [[FADD5:%.*]], %for.body ]876; CHECK-ORDERED: [[LOAD5:%.*]] = load float, ptr877; CHECK-ORDERED: [[FADD5]] = fadd nnan float [[LOAD5]], [[SUM_07]]878; CHECK-ORDERED: for.end:879; CHECK-ORDERED: [[RES:%.*]] = phi float [ [[FADD5]], %for.body ], [ [[FADD4]], %middle.block ]880; CHECK-ORDERED: ret float [[RES]]881 882; CHECK-UNORDERED-LABEL: @fadd_scalar_vf_fmf883; CHECK-UNORDERED: vector.body:884; CHECK-UNORDERED: [[VEC_PHI1:%.*]] = phi float [ 0.000000e+00, %vector.ph ], [ [[FADD1:%.*]], %vector.body ]885; CHECK-UNORDERED: [[VEC_PHI2:%.*]] = phi float [ -0.000000e+00, %vector.ph ], [ [[FADD2:%.*]], %vector.body ]886; CHECK-UNORDERED: [[VEC_PHI3:%.*]] = phi float [ -0.000000e+00, %vector.ph ], [ [[FADD3:%.*]], %vector.body ]887; CHECK-UNORDERED: [[VEC_PHI4:%.*]] = phi float [ -0.000000e+00, %vector.ph ], [ [[FADD4:%.*]], %vector.body ]888; CHECK-UNORDERED: [[LOAD1:%.*]] = load float, ptr889; CHECK-UNORDERED: [[LOAD2:%.*]] = load float, ptr890; CHECK-UNORDERED: [[LOAD3:%.*]] = load float, ptr891; CHECK-UNORDERED: [[LOAD4:%.*]] = load float, ptr892; CHECK-UNORDERED: [[FADD1]] = fadd nnan float [[LOAD1]], [[VEC_PHI1]]893; CHECK-UNORDERED: [[FADD2]] = fadd nnan float [[LOAD2]], [[VEC_PHI2]]894; CHECK-UNORDERED: [[FADD3]] = fadd nnan float [[LOAD3]], [[VEC_PHI3]]895; CHECK-UNORDERED: [[FADD4]] = fadd nnan float [[LOAD4]], [[VEC_PHI4]]896; CHECK-UNORDERED-NOT: @llvm.vector.reduce.fadd897; CHECK-UNORDERED: middle.block:898; CHECK-UNORDERED: [[BIN_RDX1:%.*]] = fadd nnan float [[FADD2]], [[FADD1]]899; CHECK-UNORDERED: [[BIN_RDX2:%.*]] = fadd nnan float [[FADD3]], [[BIN_RDX1]]900; CHECK-UNORDERED: [[BIN_RDX3:%.*]] = fadd nnan float [[FADD4]], [[BIN_RDX2]]901; CHECK-UNORDERED: scalar.ph:902; CHECK-UNORDERED: [[MERGE_RDX:%.*]] = phi float [ [[BIN_RDX3]], %middle.block ], [ 0.000000e+00, %entry ]903; CHECK-UNORDERED: for.body:904; CHECK-UNORDERED: [[SUM_07:%.*]] = phi float [ [[MERGE_RDX]], %scalar.ph ], [ [[FADD5:%.*]], %for.body ]905; CHECK-UNORDERED: [[LOAD5:%.*]] = load float, ptr906; CHECK-UNORDERED: [[FADD5]] = fadd nnan float [[LOAD5]], [[SUM_07]]907; CHECK-UORDERED: for.end908; CHECK-UNORDERED: [[RES:%.*]] = phi float [ [[FADD5]], %for.body ], [ [[BIN_RDX3]], %middle.block ]909; CHECK-UNORDERED: ret float [[RES]]910 911; CHECK-NOT-VECTORIZED-LABEL: @fadd_scalar_vf_fmf912; CHECK-NOT-VECTORIZED-NOT: vector.body913 914entry:915 br label %for.body916 917for.body:918 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]919 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]920 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv921 %0 = load float, ptr %arrayidx, align 4922 %add = fadd nnan float %0, %sum.07923 %iv.next = add nuw nsw i64 %iv, 1924 %exitcond.not = icmp eq i64 %iv.next, %n925 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !4926 927for.end:928 ret float %add929}930 931; Test case where the reduction step is a first-order recurrence.932define double @reduction_increment_by_first_order_recurrence(i32 %n) {933; CHECK-ORDERED-LABEL: @reduction_increment_by_first_order_recurrence(934; CHECK-ORDERED: vector.body:935; CHECK-ORDERED: [[RED:%.*]] = phi double [ 0.000000e+00, %vector.ph ], [ [[RED_NEXT:%.*]], %vector.body ]936; CHECK-ORDERED: [[VECTOR_RECUR:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 0.000000e+00>, %vector.ph ], [ [[FOR_NEXT:%.*]], %vector.body ]937; CHECK-ORDERED: [[FOR_NEXT]] = sitofp <4 x i32> %vec.ind to <4 x double>938; CHECK-ORDERED: [[TMP1:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[FOR_NEXT]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>939; CHECK-ORDERED: [[RED_NEXT]] = call double @llvm.vector.reduce.fadd.v4f64(double [[RED]], <4 x double> [[TMP1]])940; CHECK-ORDERED: scalar.ph:941; CHECK-ORDERED: = phi double [ [[RED_NEXT]], %middle.block ]942;943; CHECK-UNORDERED-LABEL: @reduction_increment_by_first_order_recurrence(944; CHECK-UNORDERED: vector.body:945; CHECK-UNORDERED: [[RED:%.*]] = phi <4 x double> [ <double 0.000000e+00, double -0.000000e+00, double -0.000000e+00, double -0.000000e+00>, %vector.ph ], [ [[RED_NEXT:%.*]], %vector.body ]946; CHECK-UNORDERED: [[VECTOR_RECUR:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 0.000000e+00>, %vector.ph ], [ [[FOR_NEXT:%.*]], %vector.body ]947; CHECK-UNORDERED: [[FOR_NEXT]] = sitofp <4 x i32> %vec.ind to <4 x double>948; CHECK-UNORDERED: [[TMP1:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[FOR_NEXT]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>949; CHECK-UNORDERED: [[RED_NEXT]] = fadd <4 x double> [[TMP1]], [[RED]]950; CHECK-UNORDERED: middle.block:951; CHECK-UNORDERED: [[RDX:%.*]] = call double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[RED_NEXT]])952; CHECK-UNORDERED: scalar.ph:953; CHECK-UNORDERED: [[BC_MERGE_RDX:%.*]] = phi double [ [[RDX]], %middle.block ]954;955; CHECK-NOT-VECTORIZED-LABEL: @reduction_increment_by_first_order_recurrence(956; CHECK-NOT-VECTORIZED-NOT: vector.body957;958entry:959 br label %loop960 961loop:962 %red = phi double [ 0.0, %entry ], [ %red.next, %loop ]963 %for = phi double [ 0.0, %entry ], [ %for.next, %loop ]964 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]965 %red.next = fadd double %for, %red966 %for.next = sitofp i32 %iv to double967 %iv.next = add nsw i32 %iv, 1968 %ec = icmp eq i32 %iv.next, %n969 br i1 %ec, label %exit, label %loop, !llvm.loop !13970 971exit:972 %res = phi double [ %red.next, %loop ]973 ret double %res974}975 976; We should not mark the fadd as an ordered reduction here as there are977; more than 2 uses of the instruction978define float @fadd_multiple_use(i64 %n) {979; CHECK-ORDERED-LABEL: @fadd_multiple_use980; CHECK-ORDERED-LABEL-NOT: vector.body981 982; CHECK-UNORDERED-LABEL: @fadd_multiple_use983; CHECK-UNORDERED-LABEL-NOT: vector.body984 985; CHECK-NOT-VECTORIZED-LABEL: @fadd_multiple_use986; CHECK-NOT-VECTORIZED-NOT: vector.body987 988entry:989 br label %for.body990 991for.body:992 %iv = phi i64 [ 0, %entry ], [ %iv.next2, %bb2 ]993 %red = phi float [ 0.0, %entry ], [ %fadd, %bb2 ]994 %phi1 = phi i64 [ 0, %entry ], [ %iv.next, %bb2 ]995 %fadd = fadd float %red, 1.000000e+00996 %iv.next = add nsw i64 %phi1, 1997 %cmp = icmp ult i64 %iv, %n998 br i1 %cmp, label %bb2, label %bb1999 1000bb1:1001 %phi2 = phi float [ %fadd, %for.body ]1002 ret float %phi21003 1004bb2:1005 %iv.next2 = add nuw nsw i64 %iv, 11006 br i1 false, label %for.end, label %for.body1007 1008for.end:1009 %phi3 = phi float [ %fadd, %bb2 ]1010 ret float %phi31011}1012 1013; Test case where the loop has a call to the llvm.fmuladd intrinsic.1014define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) {1015; CHECK-ORDERED-LABEL: @fmuladd_strict1016; CHECK-ORDERED: vector.body:1017; CHECK-ORDERED: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %vector.ph ], [ [[RDX3:%.*]], %vector.body ]1018; CHECK-ORDERED: [[WIDE_LOAD:%.*]] = load <8 x float>, ptr1019; CHECK-ORDERED: [[WIDE_LOAD1:%.*]] = load <8 x float>, ptr1020; CHECK-ORDERED: [[WIDE_LOAD2:%.*]] = load <8 x float>, ptr1021; CHECK-ORDERED: [[WIDE_LOAD3:%.*]] = load <8 x float>, ptr1022; CHECK-ORDERED: [[WIDE_LOAD4:%.*]] = load <8 x float>, ptr1023; CHECK-ORDERED: [[WIDE_LOAD5:%.*]] = load <8 x float>, ptr1024; CHECK-ORDERED: [[WIDE_LOAD6:%.*]] = load <8 x float>, ptr1025; CHECK-ORDERED: [[WIDE_LOAD7:%.*]] = load <8 x float>, ptr1026; CHECK-ORDERED: [[FMUL:%.*]] = fmul <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD4]]1027; CHECK-ORDERED: [[FMUL1:%.*]] = fmul <8 x float> [[WIDE_LOAD1]], [[WIDE_LOAD5]]1028; CHECK-ORDERED: [[FMUL2:%.*]] = fmul <8 x float> [[WIDE_LOAD2]], [[WIDE_LOAD6]]1029; CHECK-ORDERED: [[FMUL3:%.*]] = fmul <8 x float> [[WIDE_LOAD3]], [[WIDE_LOAD7]]1030; CHECK-ORDERED: [[RDX:%.*]] = call float @llvm.vector.reduce.fadd.v8f32(float [[VEC_PHI]], <8 x float> [[FMUL]])1031; CHECK-ORDERED: [[RDX1:%.*]] = call float @llvm.vector.reduce.fadd.v8f32(float [[RDX]], <8 x float> [[FMUL1]])1032; CHECK-ORDERED: [[RDX2:%.*]] = call float @llvm.vector.reduce.fadd.v8f32(float [[RDX1]], <8 x float> [[FMUL2]])1033; CHECK-ORDERED: [[RDX3]] = call float @llvm.vector.reduce.fadd.v8f32(float [[RDX2]], <8 x float> [[FMUL3]])1034; CHECK-ORDERED: for.body:1035; CHECK-ORDERED: [[SUM_07:%.*]] = phi float [ {{.*}}, %scalar.ph ], [ [[MULADD:%.*]], %for.body ]1036; CHECK-ORDERED: [[LOAD:%.*]] = load float, ptr1037; CHECK-ORDERED: [[LOAD1:%.*]] = load float, ptr1038; CHECK-ORDERED: [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[LOAD]], float [[LOAD1]], float [[SUM_07]])1039; CHECK-ORDERED: for.end1040; CHECK-ORDERED: [[RES:%.*]] = phi float [ [[MULADD]], %for.body ], [ [[RDX3]], %middle.block ]1041 1042; CHECK-UNORDERED-LABEL: @fmuladd_strict1043; CHECK-UNORDERED: vector.body:1044; CHECK-UNORDERED: [[VEC_PHI:%.*]] = phi <8 x float> [ <float 0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %vector.ph ], [ [[FMULADD:%.*]], %vector.body ]1045; CHECK-UNORDERED: [[WIDE_LOAD:%.*]] = load <8 x float>, ptr1046; CHECK-UNORDERED: [[WIDE_LOAD1:%.*]] = load <8 x float>, ptr1047; CHECK-UNORDERED: [[WIDE_LOAD2:%.*]] = load <8 x float>, ptr1048; CHECK-UNORDERED: [[WIDE_LOAD3:%.*]] = load <8 x float>, ptr1049; CHECK-UNORDERED: [[WIDE_LOAD4:%.*]] = load <8 x float>, ptr1050; CHECK-UNORDERED: [[FMULADD]] = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> [[WIDE_LOAD]], <8 x float> [[WIDE_LOAD4]], <8 x float> [[VEC_PHI]])1051; CHECK-UNORDERED-NOT: llvm.vector.reduce.fadd1052; CHECK-UNORDERED: middle.block:1053; CHECK-UNORDERED: [[BIN_RDX1:%.*]] = fadd <8 x float>1054; CHECK-UNORDERED: [[BIN_RDX2:%.*]] = fadd <8 x float>1055; CHECK-UNORDERED: [[BIN_RDX3:%.*]] = fadd <8 x float>1056; CHECK-UNORDERED: [[RDX:%.*]] = call float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[BIN_RDX3]])1057; CHECK-UNORDERED: for.body:1058; CHECK-UNORDERED: [[SUM_07:%.*]] = phi float [ {{.*}}, %scalar.ph ], [ [[MULADD:%.*]], %for.body ]1059; CHECK-UNORDERED: [[LOAD:%.*]] = load float, ptr1060; CHECK-UNORDERED: [[LOAD2:%.*]] = load float, ptr1061; CHECK-UNORDERED: [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[LOAD]], float [[LOAD2]], float [[SUM_07]])1062; CHECK-UNORDERED: for.end:1063; CHECK-UNORDERED: [[RES:%.*]] = phi float [ [[MULADD]], %for.body ], [ [[RDX]], %middle.block ]1064; CHECK-UNORDERED: ret float [[RES]]1065 1066; CHECK-NOT-VECTORIZED-LABEL: @fmuladd_strict1067; CHECK-NOT-VECTORIZED-NOT: vector.body1068 1069entry:1070 br label %for.body1071 1072for.body:1073 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]1074 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd, %for.body ]1075 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv1076 %0 = load float, ptr %arrayidx, align 41077 %arrayidx2 = getelementptr inbounds float, ptr %b, i64 %iv1078 %1 = load float, ptr %arrayidx2, align 41079 %muladd = tail call float @llvm.fmuladd.f32(float %0, float %1, float %sum.07)1080 %iv.next = add nuw nsw i64 %iv, 11081 %exitcond.not = icmp eq i64 %iv.next, %n1082 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !11083 1084for.end:1085 ret float %muladd1086}1087 1088; Test reductions for a VF of 1 and a UF > 1 where the loop has a call to the llvm.fmuladd intrinsic.1089define float @fmuladd_scalar_vf(ptr %a, ptr %b, i64 %n) {1090; CHECK-ORDERED-LABEL: @fmuladd_scalar_vf1091; CHECK-ORDERED: vector.body:1092; CHECK-ORDERED: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %vector.ph ], [ [[FADD3:%.*]], %vector.body ]1093; CHECK-ORDERED: [[LOAD:%.*]] = load float, ptr1094; CHECK-ORDERED: [[LOAD1:%.*]] = load float, ptr1095; CHECK-ORDERED: [[LOAD2:%.*]] = load float, ptr1096; CHECK-ORDERED: [[LOAD3:%.*]] = load float, ptr1097; CHECK-ORDERED: [[LOAD4:%.*]] = load float, ptr1098; CHECK-ORDERED: [[LOAD5:%.*]] = load float, ptr1099; CHECK-ORDERED: [[LOAD6:%.*]] = load float, ptr1100; CHECK-ORDERED: [[LOAD7:%.*]] = load float, ptr1101; CHECK-ORDERED: [[FMUL:%.*]] = fmul float [[LOAD]], [[LOAD4]]1102; CHECK-ORDERED: [[FMUL1:%.*]] = fmul float [[LOAD1]], [[LOAD5]]1103; CHECK-ORDERED: [[FMUL2:%.*]] = fmul float [[LOAD2]], [[LOAD6]]1104; CHECK-ORDERED: [[FMUL3:%.*]] = fmul float [[LOAD3]], [[LOAD7]]1105; CHECK-ORDERED: [[FADD:%.*]] = fadd float [[VEC_PHI]], [[FMUL]]1106; CHECK-ORDERED: [[FADD1:%.*]] = fadd float [[FADD]], [[FMUL1]]1107; CHECK-ORDERED: [[FADD2:%.*]] = fadd float [[FADD1]], [[FMUL2]]1108; CHECK-ORDERED: [[FADD3]] = fadd float [[FADD2]], [[FMUL3]]1109; CHECK-ORDERED-NOT: llvm.vector.reduce.fadd1110; CHECK-ORDERED: scalar.ph1111; CHECK-ORDERED: [[MERGE_RDX:%.*]] = phi float [ [[FADD3]], %middle.block ], [ 0.000000e+00, %entry ]1112; CHECK-ORDERED: for.body1113; CHECK-ORDERED: [[SUM_07:%.*]] = phi float [ [[MERGE_RDX]], %scalar.ph ], [ [[MULADD:%.*]], %for.body ]1114; CHECK-ORDERED: [[LOAD8:%.*]] = load float, ptr1115; CHECK-ORDERED: [[LOAD9:%.*]] = load float, ptr1116; CHECK-ORDERED: [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[LOAD8]], float [[LOAD9]], float [[SUM_07]])1117; CHECK-ORDERED: for.end1118; CHECK-ORDERED: [[RES:%.*]] = phi float [ [[MULADD]], %for.body ], [ [[FADD3]], %middle.block ]1119; CHECK-ORDERED: ret float [[RES]]1120 1121; CHECK-UNORDERED-LABEL: @fmuladd_scalar_vf1122; CHECK-UNORDERED: vector.body:1123; CHECK-UNORDERED: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %vector.ph ], [ [[FMULADD:%.*]], %vector.body ]1124; CHECK-UNORDERED: [[VEC_PHI1:%.*]] = phi float [ -0.000000e+00, %vector.ph ], [ [[FMULADD1:%.*]], %vector.body ]1125; CHECK-UNORDERED: [[VEC_PHI2:%.*]] = phi float [ -0.000000e+00, %vector.ph ], [ [[FMULADD2:%.*]], %vector.body ]1126; CHECK-UNORDERED: [[VEC_PHI3:%.*]] = phi float [ -0.000000e+00, %vector.ph ], [ [[FMULADD3:%.*]], %vector.body ]1127; CHECK-UNORDERED: [[LOAD:%.*]] = load float, ptr1128; CHECK-UNORDERED: [[LOAD1:%.*]] = load float, ptr1129; CHECK-UNORDERED: [[LOAD2:%.*]] = load float, ptr1130; CHECK-UNORDERED: [[LOAD3:%.*]] = load float, ptr1131; CHECK-UNORDERED: [[LOAD4:%.*]] = load float, ptr1132; CHECK-UNORDERED: [[LOAD5:%.*]] = load float, ptr1133; CHECK-UNORDERED: [[LOAD6:%.*]] = load float, ptr1134; CHECK-UNORDERED: [[LOAD7:%.*]] = load float, ptr1135; CHECK-UNORDERED: [[FMULADD]] = tail call float @llvm.fmuladd.f32(float [[LOAD]], float [[LOAD4]], float [[VEC_PHI]])1136; CHECK-UNORDERED: [[FMULADD1]] = tail call float @llvm.fmuladd.f32(float [[LOAD1]], float [[LOAD5]], float [[VEC_PHI1]])1137; CHECK-UNORDERED: [[FMULADD2]] = tail call float @llvm.fmuladd.f32(float [[LOAD2]], float [[LOAD6]], float [[VEC_PHI2]])1138; CHECK-UNORDERED: [[FMULADD3]] = tail call float @llvm.fmuladd.f32(float [[LOAD3]], float [[LOAD7]], float [[VEC_PHI3]])1139; CHECK-UNORDERED-NOT: llvm.vector.reduce.fadd1140; CHECK-UNORDERED: middle.block:1141; CHECK-UNORDERED: [[BIN_RDX:%.*]] = fadd float [[FMULADD1]], [[FMULADD]]1142; CHECK-UNORDERED: [[BIN_RDX1:%.*]] = fadd float [[FMULADD2]], [[BIN_RDX]]1143; CHECK-UNORDERED: [[BIN_RDX2:%.*]] = fadd float [[FMULADD3]], [[BIN_RDX1]]1144; CHECK-UNORDERED: scalar.ph:1145; CHECK-UNORDERED: [[MERGE_RDX:%.*]] = phi float [ [[BIN_RDX2]], %middle.block ], [ 0.000000e+00, %entry ]1146; CHECK-UNORDERED: for.body:1147; CHECK-UNORDERED: [[SUM_07:%.*]] = phi float [ [[MERGE_RDX]], %scalar.ph ], [ [[MULADD:%.*]], %for.body ]1148; CHECK-UNORDERED: [[LOAD8:%.*]] = load float, ptr1149; CHECK-UNORDERED: [[LOAD9:%.*]] = load float, ptr1150; CHECK-UNORDERED: [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[LOAD8]], float [[LOAD9]], float [[SUM_07]])1151; CHECK-UNORDERED: for.end:1152; CHECK-UNORDERED: [[RES:%.*]] = phi float [ [[MULADD]], %for.body ], [ [[BIN_RDX2]], %middle.block ]1153; CHECK-UNORDERED: ret float [[RES]]1154 1155; CHECK-NOT-VECTORIZED-LABEL: @fmuladd_scalar_vf1156; CHECK-NOT-VECTORIZED-NOT: vector.body1157 1158entry:1159 br label %for.body1160 1161for.body:1162 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]1163 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd, %for.body ]1164 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv1165 %0 = load float, ptr %arrayidx, align 41166 %arrayidx2 = getelementptr inbounds float, ptr %b, i64 %iv1167 %1 = load float, ptr %arrayidx2, align 41168 %muladd = tail call float @llvm.fmuladd.f32(float %0, float %1, float %sum.07)1169 %iv.next = add nuw nsw i64 %iv, 11170 %exitcond.not = icmp eq i64 %iv.next, %n1171 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !41172 1173for.end:1174 ret float %muladd1175}1176 1177; Test case where the reduction phi is one of the mul operands of the fmuladd.1178define float @fmuladd_phi_is_mul_operand(ptr %a, ptr %b, i64 %n) {1179; CHECK-ORDERED-LABEL: @fmuladd_phi_is_mul_operand1180; CHECK-ORDERED-NOT: vector.body1181 1182; CHECK-UNORDERED-LABEL: @fmuladd_phi_is_mul_operand1183; CHECK-UNORDERED-NOT: vector.body1184 1185; CHECK-NOT-VECTORIZED-LABEL: @fmuladd_phi_is_mul_operand1186; CHECK-NOT-VECTORIZED-NOT: vector.body1187 1188entry:1189 br label %for.body1190 1191for.body:1192 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]1193 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd, %for.body ]1194 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv1195 %0 = load float, ptr %arrayidx, align 41196 %arrayidx2 = getelementptr inbounds float, ptr %b, i64 %iv1197 %1 = load float, ptr %arrayidx2, align 41198 %muladd = tail call float @llvm.fmuladd.f32(float %sum.07, float %0, float %1)1199 %iv.next = add nuw nsw i64 %iv, 11200 %exitcond.not = icmp eq i64 %iv.next, %n1201 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !11202 1203for.end:1204 ret float %muladd1205}1206 1207; Test case where the reduction phi is two operands of the fmuladd.1208define float @fmuladd_phi_is_two_operands(ptr %a, i64 %n) {1209; CHECK-ORDERED-LABEL: @fmuladd_phi_is_two_operands1210; CHECK-ORDERED-NOT: vector.body1211 1212; CHECK-UNORDERED-LABEL: @fmuladd_phi_is_two_operands1213; CHECK-UNORDERED-NOT: vector.body1214 1215; CHECK-NOT-VECTORIZED-LABEL: @fmuladd_phi_is_two_operands1216; CHECK-NOT-VECTORIZED-NOT: vector.body1217 1218entry:1219 br label %for.body1220 1221for.body:1222 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]1223 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd, %for.body ]1224 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv1225 %0 = load float, ptr %arrayidx, align 41226 %muladd = tail call float @llvm.fmuladd.f32(float %sum.07, float %0, float %sum.07)1227 %iv.next = add nuw nsw i64 %iv, 11228 %exitcond.not = icmp eq i64 %iv.next, %n1229 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !11230 1231for.end:1232 ret float %muladd1233}1234 1235; Test case with multiple calls to llvm.fmuladd, which is not safe to reorder1236; so is only vectorized in the unordered (fast) case.1237define float @fmuladd_multiple(ptr %a, ptr %b, i64 %n) {1238; CHECK-ORDERED-LABEL: @fmuladd_multiple1239; CHECK-ORDERED-NOT: vector.body:1240 1241; CHECK-UNORDERED-LABEL: @fmuladd_multiple1242; CHECK-UNORDERED: vector.body:1243; CHECK-UNORDERED: [[VEC_PHI:%.*]] = phi <8 x float> [ <float 0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %vector.ph ], [ [[FMULADD2:%.*]], %vector.body ]1244; CHECK-UNORDERED: [[WIDE_LOAD:%.*]] = load <8 x float>, ptr1245; CHECK-UNORDERED: [[WIDE_LOAD1:%.*]] = load <8 x float>, ptr1246; CHECK-UNORDERED: [[WIDE_LOAD2:%.*]] = load <8 x float>, ptr1247; CHECK-UNORDERED: [[WIDE_LOAD3:%.*]] = load <8 x float>, ptr1248; CHECK-UNORDERED: [[WIDE_LOAD4:%.*]] = load <8 x float>, ptr1249; CHECK-UNORDERED: [[FMULADD:%.*]] = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> [[WIDE_LOAD]], <8 x float> [[WIDE_LOAD4]], <8 x float> [[VEC_PHI]])1250; CHECK-UNORDERED: [[FMULADD2]] = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> [[WIDE_LOAD]], <8 x float> [[WIDE_LOAD4]], <8 x float> [[FMULADD]])1251; CHECK-UNORDERED-NOT: llvm.vector.reduce.fadd1252; CHECK-UNORDERED: middle.block:1253; CHECK-UNORDERED: [[BIN_RDX1:%.*]] = fadd <8 x float>1254; CHECK-UNORDERED: [[BIN_RDX2:%.*]] = fadd <8 x float>1255; CHECK-UNORDERED: [[BIN_RDX3:%.*]] = fadd <8 x float>1256; CHECK-UNORDERED: [[RDX:%.*]] = call float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[BIN_RDX3]])1257; CHECK-UNORDERED: for.body:1258; CHECK-UNORDERED: [[SUM_07:%.*]] = phi float [ {{.*}}, %scalar.ph ], [ [[MULADD2:%.*]], %for.body ]1259; CHECK-UNORDERED: [[LOAD:%.*]] = load float, ptr1260; CHECK-UNORDERED: [[LOAD2:%.*]] = load float, ptr1261; CHECK-UNORDERED: [[MULADD:%.*]] = tail call float @llvm.fmuladd.f32(float [[LOAD]], float [[LOAD2]], float [[SUM_07]])1262; CHECK-UNORDERED: [[MULADD2]] = tail call float @llvm.fmuladd.f32(float [[LOAD]], float [[LOAD2]], float [[MULADD]])1263; CHECK-UNORDERED: for.end:1264; CHECK-UNORDERED: [[RES:%.*]] = phi float [ [[MULADD2]], %for.body ], [ [[RDX]], %middle.block ]1265; CHECK-UNORDERED: ret float [[RES]]1266 1267; CHECK-NOT-VECTORIZED-LABEL: @fmuladd_multiple1268; CHECK-NOT-VECTORIZED-NOT: vector.body:1269 1270entry:1271 br label %for.body1272 1273for.body:1274 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]1275 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd2, %for.body ]1276 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv1277 %0 = load float, ptr %arrayidx, align 41278 %arrayidx2 = getelementptr inbounds float, ptr %b, i64 %iv1279 %1 = load float, ptr %arrayidx2, align 41280 %muladd = tail call float @llvm.fmuladd.f32(float %0, float %1, float %sum.07)1281 %muladd2 = tail call float @llvm.fmuladd.f32(float %0, float %1, float %muladd)1282 %iv.next = add nuw nsw i64 %iv, 11283 %exitcond.not = icmp eq i64 %iv.next, %n1284 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !11285 1286for.end:1287 ret float %muladd21288}1289 1290; Same as above but the first fmuladd is one of the mul operands of the second fmuladd.1291define float @multiple_fmuladds_mul_operand(ptr %a, ptr %b, i64 %n) {1292; CHECK-ORDERED-LABEL: @multiple_fmuladds_mul_operand1293; CHECK-ORDERED-NOT: vector.body1294 1295; CHECK-UNORDERED-LABEL: @multiple_fmuladds_mul_operand1296; CHECK-UNORDERED-NOT: vector.body1297 1298; CHECK-NOT-VECTORIZED-LABEL: @multiple_fmuladds_mul_operand1299; CHECK-NOT-VECTORIZED-NOT: vector.body1300 1301entry:1302 br label %for.body1303 1304for.body:1305 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]1306 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd2, %for.body ]1307 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv1308 %0 = load float, ptr %arrayidx, align 41309 %arrayidx2 = getelementptr inbounds float, ptr %b, i64 %iv1310 %1 = load float, ptr %arrayidx2, align 41311 %muladd = tail call float @llvm.fmuladd.f32(float %0, float %1, float %sum.07)1312 %muladd2 = tail call float @llvm.fmuladd.f32(float %0, float %muladd, float %1)1313 %iv.next = add nuw nsw i64 %iv, 11314 %exitcond.not = icmp eq i64 %iv.next, %n1315 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !11316 1317for.end:1318 ret float %muladd21319}1320 1321; Same as above but the first fmuladd is two of the operands of the second fmuladd.1322define float @multiple_fmuladds_two_operands(ptr %a, ptr %b, i64 %n) {1323; CHECK-ORDERED-LABEL: @multiple_fmuladds_two_operands1324; CHECK-ORDERED-NOT: vector.body1325 1326; CHECK-UNORDERED-LABEL: @multiple_fmuladds_two_operands1327; CHECK-UNORDERED-NOT: vector.body1328 1329; CHECK-NOT-VECTORIZED-LABEL: @multiple_fmuladds_two_operands1330; CHECK-NOT-VECTORIZED-NOT: vector.body1331 1332entry:1333 br label %for.body1334 1335for.body:1336 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]1337 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd2, %for.body ]1338 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv1339 %0 = load float, ptr %arrayidx, align 41340 %arrayidx2 = getelementptr inbounds float, ptr %b, i64 %iv1341 %1 = load float, ptr %arrayidx2, align 41342 %muladd = tail call float @llvm.fmuladd.f32(float %0, float %1, float %sum.07)1343 %muladd2 = tail call float @llvm.fmuladd.f32(float %0, float %muladd, float %muladd)1344 %iv.next = add nuw nsw i64 %iv, 11345 %exitcond.not = icmp eq i64 %iv.next, %n1346 br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !11347 1348for.end:1349 ret float %muladd21350}1351 1352declare float @llvm.fmuladd.f32(float, float, float)1353 1354; Test case with invariant store where fadd is strict.1355define void @reduction_store_to_invariant_address(ptr %dst, ptr readonly %src) {1356; CHECK-ORDERED-LABEL: @reduction_store_to_invariant_address(1357; CHECK-ORDERED: entry1358; CHECK-ORDERED: %[[DEST_PTR:.*]] = getelementptr inbounds float, ptr %dst, i64 421359; CHECK-ORDERED: vector.body1360; CHECK-ORDERED: %[[VEC_PHI:.*]] = phi float [ 0.000000e+00, %vector.ph ], [ %[[RDX:.*]], %vector.body ]1361; CHECK-ORDERED: %[[LOAD_VEC:.*]] = load <8 x float>, ptr1362; CHECK-ORDERED: %[[RDX:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float %[[VEC_PHI]], <8 x float> %[[LOAD_VEC]])1363; CHECK-ORDERED: middle.block1364; CHECK-ORDERED: store float %[[RDX]], ptr %[[DEST_PTR]]1365; CHECK-ORDERED: for.body1366; CHECK-ORDERED: %[[LOAD:.*]] = load float, ptr1367; CHECK-ORDERED: %[[FADD:.*]] = fadd float %{{.*}}, %[[LOAD]]1368; CHECK-ORDERED: store float %[[FADD]], ptr %[[DEST_PTR]]1369 1370; CHECK-UNORDERED-LABEL: @reduction_store_to_invariant_address(1371; CHECK-UNORDERED: entry1372; CHECK-UNORDERED: %[[DEST_PTR:.*]] = getelementptr inbounds float, ptr %dst, i64 421373; CHECK-UNORDERED: vector.body1374; CHECK-UNORDERED: %[[VEC_PHI:.*]] = phi <8 x float> [ <float 0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %vector.ph ], [ %[[FADD_VEC:.*]], %vector.body ]1375; CHECK-UNORDERED: %[[LOAD_VEC:.*]] = load <8 x float>, ptr1376; CHECK-UNORDERED: %[[FADD_VEC]] = fadd <8 x float> %[[VEC_PHI]], %[[LOAD_VEC]]1377; CHECK-UNORDERED-NOT: call float @llvm.vector.reduce.fadd1378; CHECK-UNORDERED: middle.block1379; CHECK-UNORDERED: %[[RDX:.*]] = call float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> %[[FADD_VEC]])1380; CHECK-UNORDERED: store float %[[RDX]], ptr %[[DEST_PTR]]1381; CHECK-UNORDERED: for.body1382; CHECK-UNORDERED: %[[LOAD:.*]] = load float, ptr1383; CHECK-UNORDERED: %[[FADD:.*]] = fadd float {{.*}}, %[[LOAD]]1384; CHECK-UNORDERED: store float %[[FADD]], ptr %[[DEST_PTR]]1385 1386; CHECK-NOT-VECTORIZED-LABEL: @reduction_store_to_invariant_address(1387; CHECK-NOT-VECTORIZED-NOT: vector.body1388 1389entry:1390 %arrayidx = getelementptr inbounds float, ptr %dst, i64 421391 store float 0.000000e+00, ptr %arrayidx, align 41392 br label %for.body1393 1394for.body:1395 %0 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]1396 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]1397 %arrayidx1 = getelementptr inbounds float, ptr %src, i64 %indvars.iv1398 %1 = load float, ptr %arrayidx1, align 41399 %add = fadd float %0, %11400 store float %add, ptr %arrayidx, align 41401 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 11402 %exitcond = icmp eq i64 %indvars.iv.next, 10001403 br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !01404 1405for.cond.cleanup:1406 ret void1407}1408 1409!0 = distinct !{!0, !5, !9, !11}1410!1 = distinct !{!1, !5, !10, !11}1411!2 = distinct !{!2, !6, !9, !11}1412!3 = distinct !{!3, !7, !9, !11, !12}1413!4 = distinct !{!4, !8, !10, !11}1414!5 = !{!"llvm.loop.vectorize.width", i32 8}1415!6 = !{!"llvm.loop.vectorize.width", i32 4}1416!7 = !{!"llvm.loop.vectorize.width", i32 2}1417!8 = !{!"llvm.loop.vectorize.width", i32 1}1418!9 = !{!"llvm.loop.interleave.count", i32 1}1419!10 = !{!"llvm.loop.interleave.count", i32 4}1420!11 = !{!"llvm.loop.vectorize.enable", i1 true}1421!12 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}1422!13 = distinct !{!13, !6, !9, !11}1423