1338 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 32; RUN: opt < %s -passes=loop-vectorize,dce,instcombine -force-vector-interleave=1 -force-vector-width=4 -S | FileCheck %s3 4define i32 @reduction_sum(i32 %n, ptr %A, ptr %B) {5; CHECK-LABEL: define i32 @reduction_sum(6; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {7; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt i32 [[N]], 08; CHECK-NEXT: br i1 [[TMP1]], label [[DOTLR_PH_PREHEADER:%.*]], label [[DOT_CRIT_EDGE:%.*]]9; CHECK: .lr.ph.preheader:10; CHECK-NEXT: [[TMP2:%.*]] = zext nneg i32 [[N]] to i6411; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 412; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]13; CHECK: vector.ph:14; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 214748364415; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]16; CHECK: vector.body:17; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]18; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]19; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]20; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]21; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 422; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]23; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 424; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[VEC_PHI]], [[VEC_IND]]25; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[TMP5]], [[WIDE_LOAD]]26; CHECK-NEXT: [[TMP7]] = add <4 x i32> [[TMP6]], [[WIDE_LOAD1]]27; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 428; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)29; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]30; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]31; CHECK: middle.block:32; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])33; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP2]]34; CHECK-NEXT: br i1 [[CMP_N]], label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[SCALAR_PH]]35; CHECK: scalar.ph:36; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]37; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]38; CHECK-NEXT: br label [[DOTLR_PH:%.*]]39; CHECK: .lr.ph:40; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]41; CHECK-NEXT: [[SUM_02:%.*]] = phi i32 [ [[TMP17:%.*]], [[DOTLR_PH]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]42; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]43; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 444; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]45; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 446; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[INDVARS_IV]] to i3247; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[SUM_02]], [[TMP14]]48; CHECK-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], [[TMP11]]49; CHECK-NEXT: [[TMP17]] = add i32 [[TMP16]], [[TMP13]]50; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 151; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i3252; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]53; CHECK-NEXT: br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE_LOOPEXIT]], label [[DOTLR_PH]], !llvm.loop [[LOOP3:![0-9]+]]54; CHECK: ._crit_edge.loopexit:55; CHECK-NEXT: [[DOTLCSSA:%.*]] = phi i32 [ [[TMP17]], [[DOTLR_PH]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]56; CHECK-NEXT: br label [[DOT_CRIT_EDGE]]57; CHECK: ._crit_edge:58; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[DOTLCSSA]], [[DOT_CRIT_EDGE_LOOPEXIT]] ]59; CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]60;61 %1 = icmp sgt i32 %n, 062 br i1 %1, label %.lr.ph, label %._crit_edge63 64.lr.ph: ; preds = %0, %.lr.ph65 %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]66 %sum.02 = phi i32 [ %9, %.lr.ph ], [ 0, %0 ]67 %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv68 %3 = load i32, ptr %2, align 469 %4 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv70 %5 = load i32, ptr %4, align 471 %6 = trunc i64 %indvars.iv to i3272 %7 = add i32 %sum.02, %673 %8 = add i32 %7, %374 %9 = add i32 %8, %575 %indvars.iv.next = add i64 %indvars.iv, 176 %lftr.wideiv = trunc i64 %indvars.iv.next to i3277 %exitcond = icmp eq i32 %lftr.wideiv, %n78 br i1 %exitcond, label %._crit_edge, label %.lr.ph79 80._crit_edge: ; preds = %.lr.ph, %081 %sum.0.lcssa = phi i32 [ 0, %0 ], [ %9, %.lr.ph ]82 ret i32 %sum.0.lcssa83}84 85define i32 @reduction_prod(i32 %n, ptr %A, ptr %B) {86; CHECK-LABEL: define i32 @reduction_prod(87; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {88; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt i32 [[N]], 089; CHECK-NEXT: br i1 [[TMP1]], label [[DOTLR_PH_PREHEADER:%.*]], label [[DOT_CRIT_EDGE:%.*]]90; CHECK: .lr.ph.preheader:91; CHECK-NEXT: [[TMP2:%.*]] = zext nneg i32 [[N]] to i6492; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 493; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]94; CHECK: vector.ph:95; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 214748364496; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]97; CHECK: vector.body:98; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]99; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 1), [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]100; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]101; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]102; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4103; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]104; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4105; CHECK-NEXT: [[TMP5:%.*]] = mul <4 x i32> [[VEC_PHI]], [[VEC_IND]]106; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i32> [[TMP5]], [[WIDE_LOAD]]107; CHECK-NEXT: [[TMP7]] = mul <4 x i32> [[TMP6]], [[WIDE_LOAD1]]108; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4109; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)110; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]111; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]112; CHECK: middle.block:113; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP7]])114; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP2]]115; CHECK-NEXT: br i1 [[CMP_N]], label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[SCALAR_PH]]116; CHECK: scalar.ph:117; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]118; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 1, [[DOTLR_PH_PREHEADER]] ]119; CHECK-NEXT: br label [[DOTLR_PH:%.*]]120; CHECK: .lr.ph:121; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]122; CHECK-NEXT: [[PROD_02:%.*]] = phi i32 [ [[TMP17:%.*]], [[DOTLR_PH]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]123; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]124; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4125; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]126; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4127; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[INDVARS_IV]] to i32128; CHECK-NEXT: [[TMP15:%.*]] = mul i32 [[PROD_02]], [[TMP14]]129; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[TMP15]], [[TMP11]]130; CHECK-NEXT: [[TMP17]] = mul i32 [[TMP16]], [[TMP13]]131; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1132; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32133; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]134; CHECK-NEXT: br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE_LOOPEXIT]], label [[DOTLR_PH]], !llvm.loop [[LOOP5:![0-9]+]]135; CHECK: ._crit_edge.loopexit:136; CHECK-NEXT: [[DOTLCSSA:%.*]] = phi i32 [ [[TMP17]], [[DOTLR_PH]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]137; CHECK-NEXT: br label [[DOT_CRIT_EDGE]]138; CHECK: ._crit_edge:139; CHECK-NEXT: [[PROD_0_LCSSA:%.*]] = phi i32 [ 1, [[TMP0:%.*]] ], [ [[DOTLCSSA]], [[DOT_CRIT_EDGE_LOOPEXIT]] ]140; CHECK-NEXT: ret i32 [[PROD_0_LCSSA]]141;142 %1 = icmp sgt i32 %n, 0143 br i1 %1, label %.lr.ph, label %._crit_edge144 145.lr.ph: ; preds = %0, %.lr.ph146 %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]147 %prod.02 = phi i32 [ %9, %.lr.ph ], [ 1, %0 ]148 %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv149 %3 = load i32, ptr %2, align 4150 %4 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv151 %5 = load i32, ptr %4, align 4152 %6 = trunc i64 %indvars.iv to i32153 %7 = mul i32 %prod.02, %6154 %8 = mul i32 %7, %3155 %9 = mul i32 %8, %5156 %indvars.iv.next = add i64 %indvars.iv, 1157 %lftr.wideiv = trunc i64 %indvars.iv.next to i32158 %exitcond = icmp eq i32 %lftr.wideiv, %n159 br i1 %exitcond, label %._crit_edge, label %.lr.ph160 161._crit_edge: ; preds = %.lr.ph, %0162 %prod.0.lcssa = phi i32 [ 1, %0 ], [ %9, %.lr.ph ]163 ret i32 %prod.0.lcssa164}165 166define i32 @reduction_mix(i32 %n, ptr %A, ptr %B) {167; CHECK-LABEL: define i32 @reduction_mix(168; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {169; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt i32 [[N]], 0170; CHECK-NEXT: br i1 [[TMP1]], label [[DOTLR_PH_PREHEADER:%.*]], label [[DOT_CRIT_EDGE:%.*]]171; CHECK: .lr.ph.preheader:172; CHECK-NEXT: [[TMP2:%.*]] = zext nneg i32 [[N]] to i64173; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4174; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]175; CHECK: vector.ph:176; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 2147483644177; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]178; CHECK: vector.body:179; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]180; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]181; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]182; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]183; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4184; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]185; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4186; CHECK-NEXT: [[TMP5:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]187; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[VEC_PHI]], [[VEC_IND]]188; CHECK-NEXT: [[TMP7]] = add <4 x i32> [[TMP6]], [[TMP5]]189; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4190; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)191; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]192; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]193; CHECK: middle.block:194; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])195; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP2]]196; CHECK-NEXT: br i1 [[CMP_N]], label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[SCALAR_PH]]197; CHECK: scalar.ph:198; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]199; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]200; CHECK-NEXT: br label [[DOTLR_PH:%.*]]201; CHECK: .lr.ph:202; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]203; CHECK-NEXT: [[SUM_02:%.*]] = phi i32 [ [[TMP17:%.*]], [[DOTLR_PH]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]204; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]205; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4206; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]207; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4208; CHECK-NEXT: [[TMP14:%.*]] = mul nsw i32 [[TMP13]], [[TMP11]]209; CHECK-NEXT: [[TMP15:%.*]] = trunc i64 [[INDVARS_IV]] to i32210; CHECK-NEXT: [[TMP16:%.*]] = add i32 [[SUM_02]], [[TMP15]]211; CHECK-NEXT: [[TMP17]] = add i32 [[TMP16]], [[TMP14]]212; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1213; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32214; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]215; CHECK-NEXT: br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE_LOOPEXIT]], label [[DOTLR_PH]], !llvm.loop [[LOOP7:![0-9]+]]216; CHECK: ._crit_edge.loopexit:217; CHECK-NEXT: [[DOTLCSSA:%.*]] = phi i32 [ [[TMP17]], [[DOTLR_PH]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]218; CHECK-NEXT: br label [[DOT_CRIT_EDGE]]219; CHECK: ._crit_edge:220; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[DOTLCSSA]], [[DOT_CRIT_EDGE_LOOPEXIT]] ]221; CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]222;223 %1 = icmp sgt i32 %n, 0224 br i1 %1, label %.lr.ph, label %._crit_edge225 226.lr.ph: ; preds = %0, %.lr.ph227 %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]228 %sum.02 = phi i32 [ %9, %.lr.ph ], [ 0, %0 ]229 %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv230 %3 = load i32, ptr %2, align 4231 %4 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv232 %5 = load i32, ptr %4, align 4233 %6 = mul nsw i32 %5, %3234 %7 = trunc i64 %indvars.iv to i32235 %8 = add i32 %sum.02, %7236 %9 = add i32 %8, %6237 %indvars.iv.next = add i64 %indvars.iv, 1238 %lftr.wideiv = trunc i64 %indvars.iv.next to i32239 %exitcond = icmp eq i32 %lftr.wideiv, %n240 br i1 %exitcond, label %._crit_edge, label %.lr.ph241 242._crit_edge: ; preds = %.lr.ph, %0243 %sum.0.lcssa = phi i32 [ 0, %0 ], [ %9, %.lr.ph ]244 ret i32 %sum.0.lcssa245}246 247define i32 @reduction_mul(i32 %n, ptr %A, ptr %B) {248; CHECK-LABEL: define i32 @reduction_mul(249; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {250; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt i32 [[N]], 0251; CHECK-NEXT: br i1 [[TMP1]], label [[DOTLR_PH_PREHEADER:%.*]], label [[DOT_CRIT_EDGE:%.*]]252; CHECK: .lr.ph.preheader:253; CHECK-NEXT: [[TMP2:%.*]] = zext nneg i32 [[N]] to i64254; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4255; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]256; CHECK: vector.ph:257; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 2147483644258; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]259; CHECK: vector.body:260; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]261; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 19, i32 1, i32 1, i32 1>, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]262; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]263; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]264; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4265; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]266; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4267; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[VEC_IND]]268; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[TMP5]], [[WIDE_LOAD1]]269; CHECK-NEXT: [[TMP7]] = mul <4 x i32> [[TMP6]], [[VEC_PHI]]270; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4271; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)272; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]273; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]274; CHECK: middle.block:275; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP7]])276; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP2]]277; CHECK-NEXT: br i1 [[CMP_N]], label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[SCALAR_PH]]278; CHECK: scalar.ph:279; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]280; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 19, [[DOTLR_PH_PREHEADER]] ]281; CHECK-NEXT: br label [[DOTLR_PH:%.*]]282; CHECK: .lr.ph:283; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]284; CHECK-NEXT: [[SUM_02:%.*]] = phi i32 [ [[TMP17:%.*]], [[DOTLR_PH]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]285; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]286; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4287; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]288; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4289; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[INDVARS_IV]] to i32290; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[TMP11]], [[TMP14]]291; CHECK-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], [[TMP13]]292; CHECK-NEXT: [[TMP17]] = mul i32 [[TMP16]], [[SUM_02]]293; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1294; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32295; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]296; CHECK-NEXT: br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE_LOOPEXIT]], label [[DOTLR_PH]], !llvm.loop [[LOOP9:![0-9]+]]297; CHECK: ._crit_edge.loopexit:298; CHECK-NEXT: [[DOTLCSSA:%.*]] = phi i32 [ [[TMP17]], [[DOTLR_PH]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]299; CHECK-NEXT: br label [[DOT_CRIT_EDGE]]300; CHECK: ._crit_edge:301; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[DOTLCSSA]], [[DOT_CRIT_EDGE_LOOPEXIT]] ]302; CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]303;304 %1 = icmp sgt i32 %n, 0305 br i1 %1, label %.lr.ph, label %._crit_edge306 307.lr.ph: ; preds = %0, %.lr.ph308 %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]309 %sum.02 = phi i32 [ %9, %.lr.ph ], [ 19, %0 ]310 %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv311 %3 = load i32, ptr %2, align 4312 %4 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv313 %5 = load i32, ptr %4, align 4314 %6 = trunc i64 %indvars.iv to i32315 %7 = add i32 %3, %6316 %8 = add i32 %7, %5317 %9 = mul i32 %8, %sum.02318 %indvars.iv.next = add i64 %indvars.iv, 1319 %lftr.wideiv = trunc i64 %indvars.iv.next to i32320 %exitcond = icmp eq i32 %lftr.wideiv, %n321 br i1 %exitcond, label %._crit_edge, label %.lr.ph322 323._crit_edge: ; preds = %.lr.ph, %0324 %sum.0.lcssa = phi i32 [ 0, %0 ], [ %9, %.lr.ph ]325 ret i32 %sum.0.lcssa326}327 328define i32 @start_at_non_zero(ptr %in, ptr %coeff, ptr %out, i32 %n) {329; CHECK-LABEL: define i32 @start_at_non_zero(330; CHECK-SAME: ptr [[IN:%.*]], ptr [[COEFF:%.*]], ptr [[OUT:%.*]], i32 [[N:%.*]]) {331; CHECK-NEXT: entry:332; CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i32 [[N]], 0333; CHECK-NEXT: br i1 [[CMP7]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]334; CHECK: for.body.preheader:335; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64336; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4337; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]338; CHECK: vector.ph:339; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483644340; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]341; CHECK: vector.body:342; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]343; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 120, i32 0, i32 0, i32 0>, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]344; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[IN]], i64 [[INDEX]]345; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4346; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COEFF]], i64 [[INDEX]]347; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4348; CHECK-NEXT: [[TMP3:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]349; CHECK-NEXT: [[TMP4]] = add <4 x i32> [[TMP3]], [[VEC_PHI]]350; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4351; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]352; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]353; CHECK: middle.block:354; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])355; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]356; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]357; CHECK: scalar.ph:358; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]359; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 120, [[FOR_BODY_PREHEADER]] ]360; CHECK-NEXT: br label [[FOR_BODY:%.*]]361; CHECK: for.body:362; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]363; CHECK-NEXT: [[SUM_09:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]364; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[IN]], i64 [[INDVARS_IV]]365; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4366; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[COEFF]], i64 [[INDVARS_IV]]367; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4368; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[TMP8]], [[TMP7]]369; CHECK-NEXT: [[ADD]] = add nsw i32 [[MUL]], [[SUM_09]]370; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1371; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32372; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]373; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]374; CHECK: for.end.loopexit:375; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]376; CHECK-NEXT: br label [[FOR_END]]377; CHECK: for.end:378; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 120, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_END_LOOPEXIT]] ]379; CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]380;381entry:382 %cmp7 = icmp sgt i32 %n, 0383 br i1 %cmp7, label %for.body, label %for.end384 385for.body: ; preds = %entry, %for.body386 %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]387 %sum.09 = phi i32 [ %add, %for.body ], [ 120, %entry ]388 %arrayidx = getelementptr inbounds i32, ptr %in, i64 %indvars.iv389 %0 = load i32, ptr %arrayidx, align 4390 %arrayidx2 = getelementptr inbounds i32, ptr %coeff, i64 %indvars.iv391 %1 = load i32, ptr %arrayidx2, align 4392 %mul = mul nsw i32 %1, %0393 %add = add nsw i32 %mul, %sum.09394 %indvars.iv.next = add i64 %indvars.iv, 1395 %lftr.wideiv = trunc i64 %indvars.iv.next to i32396 %exitcond = icmp eq i32 %lftr.wideiv, %n397 br i1 %exitcond, label %for.end, label %for.body398 399for.end: ; preds = %for.body, %entry400 %sum.0.lcssa = phi i32 [ 120, %entry ], [ %add, %for.body ]401 ret i32 %sum.0.lcssa402}403 404define i32 @reduction_and(i32 %n, ptr %A, ptr %B) {405; CHECK-LABEL: define i32 @reduction_and(406; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {407; CHECK-NEXT: entry:408; CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i32 [[N]], 0409; CHECK-NEXT: br i1 [[CMP7]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]410; CHECK: for.body.preheader:411; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64412; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4413; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]414; CHECK: vector.ph:415; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483644416; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]417; CHECK: vector.body:418; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]419; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]420; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]421; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4422; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]423; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4424; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]425; CHECK-NEXT: [[TMP4]] = and <4 x i32> [[TMP3]], [[VEC_PHI]]426; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4427; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]428; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]429; CHECK: middle.block:430; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP4]])431; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]432; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]433; CHECK: scalar.ph:434; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]435; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ -1, [[FOR_BODY_PREHEADER]] ]436; CHECK-NEXT: br label [[FOR_BODY:%.*]]437; CHECK: for.body:438; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]439; CHECK-NEXT: [[RESULT_08:%.*]] = phi i32 [ [[AND:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]440; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]441; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4442; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]443; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4444; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP8]], [[TMP7]]445; CHECK-NEXT: [[AND]] = and i32 [[ADD]], [[RESULT_08]]446; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1447; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32448; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]449; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]450; CHECK: for.end.loopexit:451; CHECK-NEXT: [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]452; CHECK-NEXT: br label [[FOR_END]]453; CHECK: for.end:454; CHECK-NEXT: [[RESULT_0_LCSSA:%.*]] = phi i32 [ -1, [[ENTRY:%.*]] ], [ [[AND_LCSSA]], [[FOR_END_LOOPEXIT]] ]455; CHECK-NEXT: ret i32 [[RESULT_0_LCSSA]]456;457entry:458 %cmp7 = icmp sgt i32 %n, 0459 br i1 %cmp7, label %for.body, label %for.end460 461for.body: ; preds = %entry, %for.body462 %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]463 %result.08 = phi i32 [ %and, %for.body ], [ -1, %entry ]464 %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv465 %0 = load i32, ptr %arrayidx, align 4466 %arrayidx2 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv467 %1 = load i32, ptr %arrayidx2, align 4468 %add = add nsw i32 %1, %0469 %and = and i32 %add, %result.08470 %indvars.iv.next = add i64 %indvars.iv, 1471 %lftr.wideiv = trunc i64 %indvars.iv.next to i32472 %exitcond = icmp eq i32 %lftr.wideiv, %n473 br i1 %exitcond, label %for.end, label %for.body474 475for.end: ; preds = %for.body, %entry476 %result.0.lcssa = phi i32 [ -1, %entry ], [ %and, %for.body ]477 ret i32 %result.0.lcssa478}479 480define i32 @reduction_or(i32 %n, ptr %A, ptr %B) {481; CHECK-LABEL: define i32 @reduction_or(482; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {483; CHECK-NEXT: entry:484; CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i32 [[N]], 0485; CHECK-NEXT: br i1 [[CMP7]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]486; CHECK: for.body.preheader:487; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64488; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4489; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]490; CHECK: vector.ph:491; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483644492; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]493; CHECK: vector.body:494; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]495; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]496; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]497; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4498; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]499; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4500; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]501; CHECK-NEXT: [[TMP4]] = or <4 x i32> [[TMP3]], [[VEC_PHI]]502; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4503; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]504; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]505; CHECK: middle.block:506; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP4]])507; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]508; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]509; CHECK: scalar.ph:510; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]511; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]512; CHECK-NEXT: br label [[FOR_BODY:%.*]]513; CHECK: for.body:514; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]515; CHECK-NEXT: [[RESULT_08:%.*]] = phi i32 [ [[OR:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]516; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]517; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4518; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]519; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4520; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP8]], [[TMP7]]521; CHECK-NEXT: [[OR]] = or i32 [[ADD]], [[RESULT_08]]522; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1523; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32524; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]525; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]526; CHECK: for.end.loopexit:527; CHECK-NEXT: [[OR_LCSSA:%.*]] = phi i32 [ [[OR]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]528; CHECK-NEXT: br label [[FOR_END]]529; CHECK: for.end:530; CHECK-NEXT: [[RESULT_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OR_LCSSA]], [[FOR_END_LOOPEXIT]] ]531; CHECK-NEXT: ret i32 [[RESULT_0_LCSSA]]532;533entry:534 %cmp7 = icmp sgt i32 %n, 0535 br i1 %cmp7, label %for.body, label %for.end536 537for.body: ; preds = %entry, %for.body538 %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]539 %result.08 = phi i32 [ %or, %for.body ], [ 0, %entry ]540 %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv541 %0 = load i32, ptr %arrayidx, align 4542 %arrayidx2 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv543 %1 = load i32, ptr %arrayidx2, align 4544 %add = add nsw i32 %1, %0545 %or = or i32 %add, %result.08546 %indvars.iv.next = add i64 %indvars.iv, 1547 %lftr.wideiv = trunc i64 %indvars.iv.next to i32548 %exitcond = icmp eq i32 %lftr.wideiv, %n549 br i1 %exitcond, label %for.end, label %for.body550 551for.end: ; preds = %for.body, %entry552 %result.0.lcssa = phi i32 [ 0, %entry ], [ %or, %for.body ]553 ret i32 %result.0.lcssa554}555 556define i32 @reduction_xor(i32 %n, ptr %A, ptr %B) {557; CHECK-LABEL: define i32 @reduction_xor(558; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {559; CHECK-NEXT: entry:560; CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i32 [[N]], 0561; CHECK-NEXT: br i1 [[CMP7]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]562; CHECK: for.body.preheader:563; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64564; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4565; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]566; CHECK: vector.ph:567; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483644568; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]569; CHECK: vector.body:570; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]571; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]572; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]573; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4574; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]575; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4576; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]577; CHECK-NEXT: [[TMP4]] = xor <4 x i32> [[TMP3]], [[VEC_PHI]]578; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4579; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]580; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]581; CHECK: middle.block:582; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> [[TMP4]])583; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]584; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]585; CHECK: scalar.ph:586; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]587; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]588; CHECK-NEXT: br label [[FOR_BODY:%.*]]589; CHECK: for.body:590; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]591; CHECK-NEXT: [[RESULT_08:%.*]] = phi i32 [ [[XOR:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]592; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]593; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4594; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]595; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4596; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP8]], [[TMP7]]597; CHECK-NEXT: [[XOR]] = xor i32 [[ADD]], [[RESULT_08]]598; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1599; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32600; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]601; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]602; CHECK: for.end.loopexit:603; CHECK-NEXT: [[XOR_LCSSA:%.*]] = phi i32 [ [[XOR]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]604; CHECK-NEXT: br label [[FOR_END]]605; CHECK: for.end:606; CHECK-NEXT: [[RESULT_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[XOR_LCSSA]], [[FOR_END_LOOPEXIT]] ]607; CHECK-NEXT: ret i32 [[RESULT_0_LCSSA]]608;609entry:610 %cmp7 = icmp sgt i32 %n, 0611 br i1 %cmp7, label %for.body, label %for.end612 613for.body: ; preds = %entry, %for.body614 %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]615 %result.08 = phi i32 [ %xor, %for.body ], [ 0, %entry ]616 %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv617 %0 = load i32, ptr %arrayidx, align 4618 %arrayidx2 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv619 %1 = load i32, ptr %arrayidx2, align 4620 %add = add nsw i32 %1, %0621 %xor = xor i32 %add, %result.08622 %indvars.iv.next = add i64 %indvars.iv, 1623 %lftr.wideiv = trunc i64 %indvars.iv.next to i32624 %exitcond = icmp eq i32 %lftr.wideiv, %n625 br i1 %exitcond, label %for.end, label %for.body626 627for.end: ; preds = %for.body, %entry628 %result.0.lcssa = phi i32 [ 0, %entry ], [ %xor, %for.body ]629 ret i32 %result.0.lcssa630}631 632; In this code the subtracted variable is on the RHS and this is not an induction variable.633define i32 @reduction_sub_rhs(i32 %n, ptr %A) {634; CHECK-LABEL: define i32 @reduction_sub_rhs(635; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]]) {636; CHECK-NEXT: entry:637; CHECK-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0638; CHECK-NEXT: br i1 [[CMP4]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]639; CHECK: for.body.preheader:640; CHECK-NEXT: br label [[FOR_BODY:%.*]]641; CHECK: for.body:642; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]643; CHECK-NEXT: [[X_05:%.*]] = phi i32 [ [[SUB:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]644; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]645; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4646; CHECK-NEXT: [[SUB]] = sub nsw i32 [[TMP0]], [[X_05]]647; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1648; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32649; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]650; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]651; CHECK: for.end.loopexit:652; CHECK-NEXT: br label [[FOR_END]]653; CHECK: for.end:654; CHECK-NEXT: [[X_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[SUB]], [[FOR_END_LOOPEXIT]] ]655; CHECK-NEXT: ret i32 [[X_0_LCSSA]]656;657entry:658 %cmp4 = icmp sgt i32 %n, 0659 br i1 %cmp4, label %for.body, label %for.end660 661for.body: ; preds = %entry, %for.body662 %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]663 %x.05 = phi i32 [ %sub, %for.body ], [ 0, %entry ]664 %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv665 %0 = load i32, ptr %arrayidx, align 4666 %sub = sub nsw i32 %0, %x.05667 %indvars.iv.next = add i64 %indvars.iv, 1668 %lftr.wideiv = trunc i64 %indvars.iv.next to i32669 %exitcond = icmp eq i32 %lftr.wideiv, %n670 br i1 %exitcond, label %for.end, label %for.body671 672for.end: ; preds = %for.body, %entry673 %x.0.lcssa = phi i32 [ 0, %entry ], [ %sub, %for.body ]674 ret i32 %x.0.lcssa675}676 677 678; In this test the reduction variable is on the LHS and we can vectorize it.679define i32 @reduction_sub_lhs(i32 %n, ptr %A) {680; CHECK-LABEL: define i32 @reduction_sub_lhs(681; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]]) {682; CHECK-NEXT: entry:683; CHECK-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0684; CHECK-NEXT: br i1 [[CMP4]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]685; CHECK: for.body.preheader:686; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64687; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4688; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]689; CHECK: vector.ph:690; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483644691; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]692; CHECK: vector.body:693; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]694; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]695; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]696; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4697; CHECK-NEXT: [[TMP2]] = sub <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]698; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4699; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]700; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]701; CHECK: middle.block:702; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])703; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]704; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]705; CHECK: scalar.ph:706; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]707; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]708; CHECK-NEXT: br label [[FOR_BODY:%.*]]709; CHECK: for.body:710; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]711; CHECK-NEXT: [[X_05:%.*]] = phi i32 [ [[SUB:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]712; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]713; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX]], align 4714; CHECK-NEXT: [[SUB]] = sub nsw i32 [[X_05]], [[TMP5]]715; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1716; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32717; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]718; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]719; CHECK: for.end.loopexit:720; CHECK-NEXT: [[SUB_LCSSA:%.*]] = phi i32 [ [[SUB]], [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]721; CHECK-NEXT: br label [[FOR_END]]722; CHECK: for.end:723; CHECK-NEXT: [[X_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[SUB_LCSSA]], [[FOR_END_LOOPEXIT]] ]724; CHECK-NEXT: ret i32 [[X_0_LCSSA]]725;726entry:727 %cmp4 = icmp sgt i32 %n, 0728 br i1 %cmp4, label %for.body, label %for.end729 730for.body: ; preds = %entry, %for.body731 %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]732 %x.05 = phi i32 [ %sub, %for.body ], [ 0, %entry ]733 %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv734 %0 = load i32, ptr %arrayidx, align 4735 %sub = sub nsw i32 %x.05, %0736 %indvars.iv.next = add i64 %indvars.iv, 1737 %lftr.wideiv = trunc i64 %indvars.iv.next to i32738 %exitcond = icmp eq i32 %lftr.wideiv, %n739 br i1 %exitcond, label %for.end, label %for.body740 741for.end: ; preds = %for.body, %entry742 %x.0.lcssa = phi i32 [ 0, %entry ], [ %sub, %for.body ]743 ret i32 %x.0.lcssa744}745 746; We can vectorize conditional reductions with multi-input phis.747define float @reduction_conditional(ptr %A, ptr %B, ptr %C, float %S) {748; CHECK-LABEL: define float @reduction_conditional(749; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], float [[S:%.*]]) {750; CHECK-NEXT: entry:751; CHECK-NEXT: br label [[VECTOR_PH:%.*]]752; CHECK: vector.ph:753; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x float> <float poison, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, float [[S]], i64 0754; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]755; CHECK: vector.body:756; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]757; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ [[TMP0]], [[VECTOR_PH]] ], [ [[PREDPHI3:%.*]], [[VECTOR_BODY]] ]758; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]759; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4760; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]761; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP2]], align 4762; CHECK-NEXT: [[TMP3:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]763; CHECK-NEXT: [[TMP4:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD1]], splat (float 1.000000e+00)764; CHECK-NEXT: [[TMP8:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)765; CHECK-NEXT: [[TMP6:%.*]] = fcmp ule <4 x float> [[WIDE_LOAD]], splat (float 2.000000e+00)766; CHECK-NEXT: [[TMP7:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD1]]767; CHECK-NEXT: [[TMP5:%.*]] = and <4 x i1> [[TMP3]], [[TMP4]]768; CHECK-NEXT: [[TMP9:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]769; CHECK-NEXT: [[TMP10:%.*]] = and <4 x i1> [[TMP6]], [[TMP8]]770; CHECK-NEXT: [[TMP11:%.*]] = and <4 x i1> [[TMP10]], [[TMP3]]771; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP11]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]772; CHECK-NEXT: [[PREDPHI2:%.*]] = select <4 x i1> [[TMP5]], <4 x float> [[TMP9]], <4 x float> [[PREDPHI]]773; CHECK-NEXT: [[PREDPHI3]] = select <4 x i1> [[TMP3]], <4 x float> [[PREDPHI2]], <4 x float> [[VEC_PHI]]774; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4775; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128776; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]777; CHECK: middle.block:778; CHECK-NEXT: br label [[FOR_INC:%.*]]779; CHECK: for.end:780; CHECK-NEXT: [[SUM_1_LCSSA:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[PREDPHI3]])781; CHECK-NEXT: ret float [[SUM_1_LCSSA]]782;783entry:784 br label %for.body785 786for.body:787 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]788 %sum.033 = phi float [ %S, %entry ], [ %sum.1, %for.inc ]789 %arrayidx = getelementptr inbounds float, ptr %A, i64 %indvars.iv790 %0 = load float, ptr %arrayidx, align 4791 %arrayidx2 = getelementptr inbounds float, ptr %B, i64 %indvars.iv792 %1 = load float, ptr %arrayidx2, align 4793 %cmp3 = fcmp ogt float %0, %1794 br i1 %cmp3, label %if.then, label %for.inc795 796if.then:797 %cmp6 = fcmp ogt float %1, 1.000000e+00798 br i1 %cmp6, label %if.then8, label %if.else799 800if.then8:801 %add = fadd fast float %sum.033, %0802 br label %for.inc803 804if.else:805 %cmp14 = fcmp ogt float %0, 2.000000e+00806 br i1 %cmp14, label %if.then16, label %for.inc807 808if.then16:809 %add19 = fadd fast float %sum.033, %1810 br label %for.inc811 812for.inc:813 %sum.1 = phi float [ %add, %if.then8 ], [ %add19, %if.then16 ], [ %sum.033, %if.else ], [ %sum.033, %for.body ]814 %indvars.iv.next = add i64 %indvars.iv, 1815 %lftr.wideiv = trunc i64 %indvars.iv.next to i32816 %exitcond = icmp ne i32 %lftr.wideiv, 128817 br i1 %exitcond, label %for.body, label %for.end818 819for.end:820 %sum.1.lcssa = phi float [ %sum.1, %for.inc ]821 ret float %sum.1.lcssa822}823 824; We can't vectorize reductions with phi inputs from outside the reduction.825define float @noreduction_phi(ptr %A, ptr %B, ptr %C, float %S) {826; CHECK-LABEL: define float @noreduction_phi(827; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], float [[S:%.*]]) {828; CHECK-NEXT: entry:829; CHECK-NEXT: br label [[FOR_BODY:%.*]]830; CHECK: for.body:831; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]832; CHECK-NEXT: [[SUM_033:%.*]] = phi float [ [[S]], [[ENTRY]] ], [ [[SUM_1:%.*]], [[FOR_INC]] ]833; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]834; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4835; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDVARS_IV]]836; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2]], align 4837; CHECK-NEXT: [[CMP3:%.*]] = fcmp ogt float [[TMP0]], [[TMP1]]838; CHECK-NEXT: br i1 [[CMP3]], label [[IF_THEN:%.*]], label [[FOR_INC]]839; CHECK: if.then:840; CHECK-NEXT: [[CMP6:%.*]] = fcmp ogt float [[TMP1]], 1.000000e+00841; CHECK-NEXT: br i1 [[CMP6]], label [[IF_THEN8:%.*]], label [[IF_ELSE:%.*]]842; CHECK: if.then8:843; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[SUM_033]], [[TMP0]]844; CHECK-NEXT: br label [[FOR_INC]]845; CHECK: if.else:846; CHECK-NEXT: [[CMP14:%.*]] = fcmp ogt float [[TMP0]], 2.000000e+00847; CHECK-NEXT: br i1 [[CMP14]], label [[IF_THEN16:%.*]], label [[FOR_INC]]848; CHECK: if.then16:849; CHECK-NEXT: [[ADD19:%.*]] = fadd fast float [[SUM_033]], [[TMP1]]850; CHECK-NEXT: br label [[FOR_INC]]851; CHECK: for.inc:852; CHECK-NEXT: [[SUM_1]] = phi float [ [[ADD]], [[IF_THEN8]] ], [ [[ADD19]], [[IF_THEN16]] ], [ 0.000000e+00, [[IF_ELSE]] ], [ [[SUM_033]], [[FOR_BODY]] ]853; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1854; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32855; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 128856; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]]857; CHECK: for.end:858; CHECK-NEXT: ret float [[SUM_1]]859;860entry:861 br label %for.body862 863for.body:864 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]865 %sum.033 = phi float [ %S, %entry ], [ %sum.1, %for.inc ]866 %arrayidx = getelementptr inbounds float, ptr %A, i64 %indvars.iv867 %0 = load float, ptr %arrayidx, align 4868 %arrayidx2 = getelementptr inbounds float, ptr %B, i64 %indvars.iv869 %1 = load float, ptr %arrayidx2, align 4870 %cmp3 = fcmp ogt float %0, %1871 br i1 %cmp3, label %if.then, label %for.inc872 873if.then:874 %cmp6 = fcmp ogt float %1, 1.000000e+00875 br i1 %cmp6, label %if.then8, label %if.else876 877if.then8:878 %add = fadd fast float %sum.033, %0879 br label %for.inc880 881if.else:882 %cmp14 = fcmp ogt float %0, 2.000000e+00883 br i1 %cmp14, label %if.then16, label %for.inc884 885if.then16:886 %add19 = fadd fast float %sum.033, %1887 br label %for.inc888 889for.inc:890 %sum.1 = phi float [ %add, %if.then8 ], [ %add19, %if.then16 ], [ 0.000000e+00, %if.else ], [ %sum.033, %for.body ]891 %indvars.iv.next = add i64 %indvars.iv, 1892 %lftr.wideiv = trunc i64 %indvars.iv.next to i32893 %exitcond = icmp ne i32 %lftr.wideiv, 128894 br i1 %exitcond, label %for.body, label %for.end895 896for.end:897 %sum.1.lcssa = phi float [ %sum.1, %for.inc ]898 ret float %sum.1.lcssa899}900 901; We can't vectorize reductions that feed another header PHI.902define float @noredux_header_phi(ptr %A, ptr %B, ptr %C, float %S) {903; CHECK-LABEL: define float @noredux_header_phi(904; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], float [[S:%.*]]) {905; CHECK-NEXT: entry:906; CHECK-NEXT: br label [[FOR_BODY:%.*]]907; CHECK: for.body:908; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]909; CHECK-NEXT: [[SUM2_09:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[ADD1:%.*]], [[FOR_BODY]] ]910; CHECK-NEXT: [[SUM_08:%.*]] = phi float [ [[S]], [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]911; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDVARS_IV]]912; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4913; CHECK-NEXT: [[ADD]] = fadd fast float [[SUM_08]], [[TMP0]]914; CHECK-NEXT: [[ADD1]] = fadd fast float [[SUM2_09]], [[ADD]]915; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1916; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32917; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 128918; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]]919; CHECK: for.end:920; CHECK-NEXT: [[ADD2:%.*]] = fadd fast float [[ADD]], [[ADD1]]921; CHECK-NEXT: ret float [[ADD2]]922;923entry:924 br label %for.body925 926for.body:927 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]928 %sum2.09 = phi float [ 0.000000e+00, %entry ], [ %add1, %for.body ]929 %sum.08 = phi float [ %S, %entry ], [ %add, %for.body ]930 %arrayidx = getelementptr inbounds float, ptr %B, i64 %indvars.iv931 %0 = load float, ptr %arrayidx, align 4932 %add = fadd fast float %sum.08, %0933 %add1 = fadd fast float %sum2.09, %add934 %indvars.iv.next = add i64 %indvars.iv, 1935 %lftr.wideiv = trunc i64 %indvars.iv.next to i32936 %exitcond = icmp ne i32 %lftr.wideiv, 128937 br i1 %exitcond, label %for.body, label %for.end938 939for.end:940 %add1.lcssa = phi float [ %add1, %for.body ]941 %add.lcssa = phi float [ %add, %for.body ]942 %add2 = fadd fast float %add.lcssa, %add1.lcssa943 ret float %add2944}945 946 947; When vectorizing a reduction whose loop header phi value is used outside the948; loop special care must be taken. Otherwise, the reduced value feeding into the949; outside user misses a few iterations (VF-1) of the loop.950; PR16522951define i32 @phivalueredux(i32 %p) {952; CHECK-LABEL: define i32 @phivalueredux(953; CHECK-SAME: i32 [[P:%.*]]) {954; CHECK-NEXT: entry:955; CHECK-NEXT: br label [[FOR_BODY:%.*]]956; CHECK: for.body:957; CHECK-NEXT: [[T_03:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]958; CHECK-NEXT: [[P_ADDR_02:%.*]] = phi i32 [ [[P]], [[ENTRY]] ], [ [[XOR:%.*]], [[FOR_BODY]] ]959; CHECK-NEXT: [[XOR]] = xor i32 [[P_ADDR_02]], -1960; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[T_03]], 1961; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 16962; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]963; CHECK: for.end:964; CHECK-NEXT: ret i32 [[P_ADDR_02]]965;966entry:967 br label %for.body968 969for.body:970 %t.03 = phi i32 [ 0, %entry ], [ %inc, %for.body ]971 %p.addr.02 = phi i32 [ %p, %entry ], [ %xor, %for.body ]972 %xor = xor i32 %p.addr.02, -1973 %inc = add nsw i32 %t.03, 1974 %exitcond = icmp eq i32 %inc, 16975 br i1 %exitcond, label %for.end, label %for.body976 977for.end:978 ret i32 %p.addr.02979}980 981; Don't vectorize a reduction value that is not the last in a reduction cyle. We982; would loose iterations (VF-1) on the operations after that use.983; PR17498984define i32 @not_last_operation(i32 %p, i32 %val) {985; CHECK-LABEL: define i32 @not_last_operation(986; CHECK-SAME: i32 [[P:%.*]], i32 [[VAL:%.*]]) {987; CHECK-NEXT: entry:988; CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[P]], 0989; CHECK-NEXT: br label [[FOR_BODY:%.*]]990; CHECK: for.body:991; CHECK-NEXT: [[INC613_1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC6_1:%.*]], [[FOR_BODY]] ]992; CHECK-NEXT: [[INC511_1:%.*]] = phi i32 [ [[VAL]], [[ENTRY]] ], [ [[INC5_1:%.*]], [[FOR_BODY]] ]993; CHECK-NEXT: [[INC4_1:%.*]] = zext i1 [[TOBOOL]] to i32994; CHECK-NEXT: [[INC511_1_INC4_1:%.*]] = add nsw i32 [[INC511_1]], [[INC4_1]]995; CHECK-NEXT: [[INC5_1]] = add nsw i32 [[INC511_1_INC4_1]], 1996; CHECK-NEXT: [[INC6_1]] = add nuw nsw i32 [[INC613_1]], 1997; CHECK-NEXT: [[EXITCOND_1:%.*]] = icmp eq i32 [[INC6_1]], 22998; CHECK-NEXT: br i1 [[EXITCOND_1]], label [[EXIT:%.*]], label [[FOR_BODY]]999; CHECK: exit:1000; CHECK-NEXT: [[INC_2:%.*]] = add nsw i32 [[INC511_1_INC4_1]], 21001; CHECK-NEXT: ret i32 [[INC_2]]1002;1003entry:1004 %tobool = icmp eq i32 %p, 01005 br label %for.body1006 1007for.body:1008 %inc613.1 = phi i32 [ 0, %entry ], [ %inc6.1, %for.body ]1009 %inc511.1 = phi i32 [ %val, %entry ], [ %inc5.1, %for.body ]1010 %0 = zext i1 %tobool to i321011 %inc4.1 = xor i32 %0, 11012 %inc511.1.inc4.1 = add nsw i32 %inc511.1, %inc4.11013 %inc5.1 = add nsw i32 %inc511.1.inc4.1, 11014 %inc6.1 = add nsw i32 %inc613.1, 11015 %exitcond.1 = icmp eq i32 %inc6.1, 221016 br i1 %exitcond.1, label %exit, label %for.body1017 1018exit:1019 %inc.2 = add nsw i32 %inc511.1.inc4.1, 21020 ret i32 %inc.21021}1022 1023define i32 @reduction_sum_multiuse(i32 %n, ptr %A, ptr %B) {1024; CHECK-LABEL: define i32 @reduction_sum_multiuse(1025; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {1026; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt i32 [[N]], 01027; CHECK-NEXT: br i1 [[TMP1]], label [[DOTLR_PH_PREHEADER:%.*]], label [[END:%.*]]1028; CHECK: .lr.ph.preheader:1029; CHECK-NEXT: [[TMP2:%.*]] = zext nneg i32 [[N]] to i641030; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 41031; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1032; CHECK: vector.ph:1033; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], 21474836441034; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1035; CHECK: vector.body:1036; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1037; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]1038; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]1039; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]1040; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 41041; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]1042; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 41043; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[VEC_PHI]], [[VEC_IND]]1044; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[TMP5]], [[WIDE_LOAD]]1045; CHECK-NEXT: [[TMP7]] = add <4 x i32> [[TMP6]], [[WIDE_LOAD1]]1046; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 41047; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)1048; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1049; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]1050; CHECK: middle.block:1051; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])1052; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP2]]1053; CHECK-NEXT: br i1 [[CMP_N]], label [[DOT_CRIT_EDGE:%.*]], label [[SCALAR_PH]]1054; CHECK: scalar.ph:1055; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]1056; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]1057; CHECK-NEXT: br label [[DOTLR_PH:%.*]]1058; CHECK: .lr.ph:1059; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]1060; CHECK-NEXT: [[SUM_02:%.*]] = phi i32 [ [[TMP17:%.*]], [[DOTLR_PH]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]1061; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]1062; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 41063; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]1064; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 41065; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[INDVARS_IV]] to i321066; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[SUM_02]], [[TMP14]]1067; CHECK-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], [[TMP11]]1068; CHECK-NEXT: [[TMP17]] = add i32 [[TMP16]], [[TMP13]]1069; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 11070; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i321071; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]1072; CHECK-NEXT: br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]], !llvm.loop [[LOOP22:![0-9]+]]1073; CHECK: ._crit_edge:1074; CHECK-NEXT: [[SUM_COPY:%.*]] = phi i32 [ [[TMP17]], [[DOTLR_PH]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]1075; CHECK-NEXT: [[TMP18:%.*]] = shl i32 [[SUM_COPY]], 11076; CHECK-NEXT: br label [[END]]1077; CHECK: end:1078; CHECK-NEXT: [[F2:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP18]], [[DOT_CRIT_EDGE]] ]1079; CHECK-NEXT: ret i32 [[F2]]1080;1081 %1 = icmp sgt i32 %n, 01082 br i1 %1, label %.lr.ph.preheader, label %end1083.lr.ph.preheader: ; preds = %01084 br label %.lr.ph1085 1086.lr.ph: ; preds = %0, %.lr.ph1087 %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %.lr.ph.preheader ]1088 %sum.02 = phi i32 [ %9, %.lr.ph ], [ 0, %.lr.ph.preheader ]1089 %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv1090 %3 = load i32, ptr %2, align 41091 %4 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv1092 %5 = load i32, ptr %4, align 41093 %6 = trunc i64 %indvars.iv to i321094 %7 = add i32 %sum.02, %61095 %8 = add i32 %7, %31096 %9 = add i32 %8, %51097 %indvars.iv.next = add i64 %indvars.iv, 11098 %lftr.wideiv = trunc i64 %indvars.iv.next to i321099 %exitcond = icmp eq i32 %lftr.wideiv, %n1100 br i1 %exitcond, label %._crit_edge, label %.lr.ph1101 1102._crit_edge: ; preds = %.lr.ph, %01103 %sum.lcssa = phi i32 [ %9, %.lr.ph ]1104 %sum.copy = phi i32 [ %9, %.lr.ph ]1105 br label %end1106 1107end:1108 %f1 = phi i32 [ 0, %0 ], [ %sum.lcssa, %._crit_edge ]1109 %f2 = phi i32 [ 0, %0 ], [ %sum.copy, %._crit_edge ]1110 %final = add i32 %f1, %f21111 ret i32 %final1112}1113 1114; This looks like a predicated reduction, but it is a reset of the reduction1115; variable. We cannot vectorize this.1116define void @reduction_reset(i32 %N, ptr %arrayA, ptr %arrayB) {1117; CHECK-LABEL: define void @reduction_reset(1118; CHECK-SAME: i32 [[N:%.*]], ptr [[ARRAYA:%.*]], ptr [[ARRAYB:%.*]]) {1119; CHECK-NEXT: entry:1120; CHECK-NEXT: [[C4:%.*]] = icmp sgt i32 [[N]], 01121; CHECK-NEXT: br i1 [[C4]], label [[DOTLR_PH_PREHEADER:%.*]], label [[DOT_CRIT_EDGE:%.*]]1122; CHECK: .lr.ph.preheader:1123; CHECK-NEXT: [[C5:%.*]] = add nsw i32 [[N]], -11124; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i641125; CHECK-NEXT: br label [[DOTLR_PH:%.*]]1126; CHECK: .lr.ph:1127; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[DOTLR_PH_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ]1128; CHECK-NEXT: [[DOT017:%.*]] = phi i32 [ 100, [[DOTLR_PH_PREHEADER]] ], [ [[CSEL:%.*]], [[DOTLR_PH]] ]1129; CHECK-NEXT: [[C6:%.*]] = getelementptr inbounds nuw i32, ptr [[ARRAYA]], i64 [[INDVARS_IV]]1130; CHECK-NEXT: [[C7:%.*]] = load i32, ptr [[C6]], align 41131; CHECK-NEXT: [[C8:%.*]] = icmp sgt i32 [[C7]], 01132; CHECK-NEXT: [[C9:%.*]] = add nsw i32 [[C7]], [[DOT017]]1133; CHECK-NEXT: [[CSEL]] = select i1 [[C8]], i32 [[C9]], i32 01134; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 11135; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]1136; CHECK-NEXT: br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[DOTLR_PH]]1137; CHECK: ._crit_edge.loopexit:1138; CHECK-NEXT: [[PHITMP19:%.*]] = sext i32 [[C5]] to i641139; CHECK-NEXT: br label [[DOT_CRIT_EDGE]]1140; CHECK: ._crit_edge:1141; CHECK-NEXT: [[DOT015_LCSSA:%.*]] = phi i64 [ -1, [[ENTRY:%.*]] ], [ [[PHITMP19]], [[DOT_CRIT_EDGE_LOOPEXIT]] ]1142; CHECK-NEXT: [[DOT0_LCSSA:%.*]] = phi i32 [ 100, [[ENTRY]] ], [ [[CSEL]], [[DOT_CRIT_EDGE_LOOPEXIT]] ]1143; CHECK-NEXT: [[C10:%.*]] = getelementptr inbounds i32, ptr [[ARRAYB]], i64 [[DOT015_LCSSA]]1144; CHECK-NEXT: store i32 [[DOT0_LCSSA]], ptr [[C10]], align 41145; CHECK-NEXT: ret void1146;1147entry:1148 %c4 = icmp sgt i32 %N, 01149 br i1 %c4, label %.lr.ph.preheader, label %._crit_edge1150 1151.lr.ph.preheader: ; preds = %entry1152 %c5 = add i32 %N, -11153 %wide.trip.count = zext i32 %N to i641154 br label %.lr.ph1155 1156.lr.ph: ; preds = %.lr.ph, %.lr.ph.preheader1157 %indvars.iv = phi i64 [ 0, %.lr.ph.preheader ], [ %indvars.iv.next, %.lr.ph ]1158 %.017 = phi i32 [ 100, %.lr.ph.preheader ], [ %csel, %.lr.ph ]1159 %c6 = getelementptr inbounds i32, ptr %arrayA, i64 %indvars.iv1160 %c7 = load i32, ptr %c6, align 41161 %c8 = icmp sgt i32 %c7, 01162 %c9 = add nsw i32 %c7, %.0171163 %csel = select i1 %c8, i32 %c9, i32 01164 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 11165 %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count1166 br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph1167 1168._crit_edge.loopexit: ; preds = %.lr.ph1169 %csel.lcssa = phi i32 [ %csel, %.lr.ph ]1170 %phitmp19 = sext i32 %c5 to i641171 br label %._crit_edge1172 1173._crit_edge: ; preds = %._crit_edge.loopexit, %entry1174 %.015.lcssa = phi i64 [ -1, %entry ], [ %phitmp19, %._crit_edge.loopexit ]1175 %.0.lcssa = phi i32 [ 100, %entry ], [ %csel.lcssa, %._crit_edge.loopexit ]1176 %c10 = getelementptr inbounds i32, ptr %arrayB, i64 %.015.lcssa1177 store i32 %.0.lcssa, ptr %c10, align 41178 ret void1179}1180 1181; Can vectorize reduction with redundant single-operand phi input.1182define i64 @reduction_with_phi_with_one_incoming_on_backedge(i16 %n, ptr %A) {1183; CHECK-LABEL: define i64 @reduction_with_phi_with_one_incoming_on_backedge(1184; CHECK-SAME: i16 [[N:%.*]], ptr [[A:%.*]]) {1185; CHECK-NEXT: entry:1186; CHECK-NEXT: [[SMAX:%.*]] = call i16 @llvm.smax.i16(i16 [[N]], i16 2)1187; CHECK-NEXT: [[TMP0:%.*]] = add nsw i16 [[SMAX]], -11188; CHECK-NEXT: [[TMP1:%.*]] = zext nneg i16 [[TMP0]] to i321189; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i16 [[N]], 51190; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1191; CHECK: vector.ph:1192; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[TMP1]], 327641193; CHECK-NEXT: [[DOTCAST:%.*]] = trunc nuw nsw i32 [[N_VEC]] to i161194; CHECK-NEXT: [[IND_END:%.*]] = or disjoint i16 [[DOTCAST]], 11195; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1196; CHECK: vector.body:1197; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1198; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]1199; CHECK-NEXT: [[DOTCAST1:%.*]] = trunc i32 [[INDEX]] to i161200; CHECK-NEXT: [[TMP2:%.*]] = sext i16 [[DOTCAST1]] to i641201; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[A]], i64 [[TMP2]]1202; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP3]], i64 81203; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP8]], align 41204; CHECK-NEXT: [[TMP4]] = add <4 x i64> [[VEC_PHI]], [[WIDE_LOAD]]1205; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 41206; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]1207; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]1208; CHECK: middle.block:1209; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP4]])1210; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[TMP1]]1211; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]1212; CHECK: scalar.ph:1213; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]1214; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]1215; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]1216; CHECK: loop.header:1217; CHECK-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]1218; CHECK-NEXT: [[SUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[LOOP_LATCH]] ]1219; CHECK-NEXT: [[TMP7:%.*]] = sext i16 [[IV]] to i641220; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr i64, ptr [[A]], i64 [[TMP7]]1221; CHECK-NEXT: [[LV_A:%.*]] = load i64, ptr [[GEP_A]], align 41222; CHECK-NEXT: [[SUM_NEXT]] = add nsw i64 [[SUM]], [[LV_A]]1223; CHECK-NEXT: br label [[LOOP_BB:%.*]]1224; CHECK: loop.bb:1225; CHECK-NEXT: br label [[LOOP_LATCH]]1226; CHECK: loop.latch:1227; CHECK-NEXT: [[IV_NEXT]] = add nsw i16 [[IV]], 11228; CHECK-NEXT: [[COND:%.*]] = icmp slt i16 [[IV_NEXT]], [[N]]1229; CHECK-NEXT: br i1 [[COND]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP24:![0-9]+]]1230; CHECK: exit:1231; CHECK-NEXT: [[LCSSA_EXIT:%.*]] = phi i64 [ [[SUM_NEXT]], [[LOOP_LATCH]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]1232; CHECK-NEXT: ret i64 [[LCSSA_EXIT]]1233;1234entry:1235 br label %loop.header1236 1237loop.header:1238 %iv = phi i16 [ 1, %entry ], [ %iv.next, %loop.latch ]1239 %sum = phi i64 [ 0, %entry ], [ %phi.sum.next, %loop.latch ]1240 %gep.A = getelementptr i64, ptr %A, i16 %iv1241 %lv.A = load i64, ptr %gep.A1242 %sum.next = add nsw i64 %sum, %lv.A1243 br label %loop.bb1244 1245loop.bb:1246 %phi.sum.next = phi i64 [ %sum.next, %loop.header ]1247 br label %loop.latch1248 1249loop.latch:1250 %iv.next = add nsw i16 %iv, 11251 %cond = icmp slt i16 %iv.next, %n1252 br i1 %cond, label %loop.header, label %exit1253 1254exit:1255 %lcssa.exit = phi i64 [ %phi.sum.next, %loop.latch ]1256 ret i64 %lcssa.exit1257}1258 1259; Can vectorize reduction with redundant two-operand phi input.1260define i64 @reduction_with_phi_with_two_incoming_on_backedge(i16 %n, ptr %A) {1261; CHECK-LABEL: define i64 @reduction_with_phi_with_two_incoming_on_backedge(1262; CHECK-SAME: i16 [[N:%.*]], ptr [[A:%.*]]) {1263; CHECK-NEXT: entry:1264; CHECK-NEXT: [[SMAX:%.*]] = call i16 @llvm.smax.i16(i16 [[N]], i16 2)1265; CHECK-NEXT: [[TMP0:%.*]] = add nsw i16 [[SMAX]], -11266; CHECK-NEXT: [[TMP1:%.*]] = zext nneg i16 [[TMP0]] to i321267; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i16 [[N]], 51268; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]1269; CHECK: vector.ph:1270; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[TMP1]], 327641271; CHECK-NEXT: [[DOTCAST:%.*]] = trunc nuw nsw i32 [[N_VEC]] to i161272; CHECK-NEXT: [[IND_END:%.*]] = or disjoint i16 [[DOTCAST]], 11273; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1274; CHECK: vector.body:1275; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1276; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]1277; CHECK-NEXT: [[DOTCAST1:%.*]] = trunc i32 [[INDEX]] to i161278; CHECK-NEXT: [[TMP2:%.*]] = sext i16 [[DOTCAST1]] to i641279; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[A]], i64 [[TMP2]]1280; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP3]], i64 81281; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP8]], align 41282; CHECK-NEXT: [[TMP4]] = add <4 x i64> [[VEC_PHI]], [[WIDE_LOAD]]1283; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 41284; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]1285; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]1286; CHECK: middle.block:1287; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP4]])1288; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[TMP1]]1289; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]1290; CHECK: scalar.ph:1291; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]1292; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]1293; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]1294; CHECK: loop.header:1295; CHECK-NEXT: [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]1296; CHECK-NEXT: [[SUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[LOOP_LATCH]] ]1297; CHECK-NEXT: [[TMP7:%.*]] = sext i16 [[IV]] to i641298; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr i64, ptr [[A]], i64 [[TMP7]]1299; CHECK-NEXT: [[LV_A:%.*]] = load i64, ptr [[GEP_A]], align 41300; CHECK-NEXT: [[SUM_NEXT]] = add nsw i64 [[SUM]], [[LV_A]]1301; CHECK-NEXT: [[CMP_0:%.*]] = icmp eq i64 [[LV_A]], 291302; CHECK-NEXT: br i1 [[CMP_0]], label [[LOOP_BB:%.*]], label [[LOOP_LATCH]]1303; CHECK: loop.bb:1304; CHECK-NEXT: br label [[LOOP_LATCH]]1305; CHECK: loop.latch:1306; CHECK-NEXT: [[IV_NEXT]] = add nsw i16 [[IV]], 11307; CHECK-NEXT: [[COND:%.*]] = icmp slt i16 [[IV_NEXT]], [[N]]1308; CHECK-NEXT: br i1 [[COND]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]1309; CHECK: exit:1310; CHECK-NEXT: [[LCSSA_EXIT:%.*]] = phi i64 [ [[SUM_NEXT]], [[LOOP_LATCH]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]1311; CHECK-NEXT: ret i64 [[LCSSA_EXIT]]1312;1313entry:1314 br label %loop.header1315 1316loop.header:1317 %iv = phi i16 [ 1, %entry ], [ %iv.next, %loop.latch ]1318 %sum = phi i64 [ 0, %entry ], [ %phi.sum.next, %loop.latch ]1319 %gep.A = getelementptr i64, ptr %A, i16 %iv1320 %lv.A = load i64, ptr %gep.A1321 %sum.next = add nsw i64 %sum, %lv.A1322 %cmp.0 = icmp eq i64 %lv.A, 291323 br i1 %cmp.0, label %loop.bb, label %loop.latch1324 1325loop.bb:1326 br label %loop.latch1327 1328loop.latch:1329 %phi.sum.next = phi i64 [ %sum.next, %loop.bb ], [ %sum.next, %loop.header ]1330 %iv.next = add nsw i16 %iv, 11331 %cond = icmp slt i16 %iv.next, %n1332 br i1 %cond, label %loop.header, label %exit1333 1334exit:1335 %lcssa.exit = phi i64 [ %phi.sum.next, %loop.latch ]1336 ret i64 %lcssa.exit1337}1338