519 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt < %s -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -passes=loop-vectorize,dce,instcombine -tail-predication=enabled -S | FileCheck %s3 4target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"5target triple = "thumbv8.1m.main-none-none-eabi"6 7define i32 @reduction_sum_single(ptr noalias nocapture %A) {8; CHECK-LABEL: @reduction_sum_single(9; CHECK-NEXT: entry:10; CHECK-NEXT: br label [[VECTOR_PH:%.*]]11; CHECK: vector.ph:12; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]13; CHECK: vector.body:14; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]15; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]16; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 257)17; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[INDEX]]18; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> zeroinitializer)19; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_MASKED_LOAD]])20; CHECK-NEXT: [[TMP2]] = add i32 [[VEC_PHI]], [[TMP1]]21; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 422; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 26023; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]24; CHECK: middle.block:25; CHECK-NEXT: br label [[DOTLR_PH:%.*]]26; CHECK: ._crit_edge:27; CHECK-NEXT: ret i32 [[TMP2]]28;29entry:30 br label %.lr.ph31 32.lr.ph: ; preds = %entry, %.lr.ph33 %indvars.iv = phi i32 [ %indvars.iv.next, %.lr.ph ], [ 0, %entry ]34 %sum.02 = phi i32 [ %l7, %.lr.ph ], [ 0, %entry ]35 %l2 = getelementptr inbounds i32, ptr %A, i32 %indvars.iv36 %l3 = load i32, ptr %l2, align 437 %l7 = add i32 %sum.02, %l338 %indvars.iv.next = add i32 %indvars.iv, 139 %exitcond = icmp eq i32 %indvars.iv.next, 25740 br i1 %exitcond, label %._crit_edge, label %.lr.ph41 42._crit_edge: ; preds = %.lr.ph43 %sum.0.lcssa = phi i32 [ %l7, %.lr.ph ]44 ret i32 %sum.0.lcssa45}46 47define i32 @reduction_sum(ptr noalias nocapture %A, ptr noalias nocapture %B) {48; CHECK-LABEL: @reduction_sum(49; CHECK-NEXT: entry:50; CHECK-NEXT: br label [[VECTOR_PH:%.*]]51; CHECK: vector.ph:52; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]53; CHECK: vector.body:54; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]55; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]56; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ]57; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 257)58; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[INDEX]]59; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> zeroinitializer)60; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i32 [[INDEX]]61; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> zeroinitializer)62; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[VEC_IND]], <4 x i32> zeroinitializer63; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])64; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[VEC_PHI]], [[TMP3]]65; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_MASKED_LOAD]])66; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[TMP4]], [[TMP5]]67; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_MASKED_LOAD1]])68; CHECK-NEXT: [[TMP8]] = add i32 [[TMP6]], [[TMP7]]69; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 470; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)71; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 26072; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]73; CHECK: middle.block:74; CHECK-NEXT: br label [[DOTLR_PH:%.*]]75; CHECK: ._crit_edge:76; CHECK-NEXT: ret i32 [[TMP8]]77;78entry:79 br label %.lr.ph80 81.lr.ph: ; preds = %entry, %.lr.ph82 %indvars.iv = phi i32 [ %indvars.iv.next, %.lr.ph ], [ 0, %entry ]83 %sum.02 = phi i32 [ %l9, %.lr.ph ], [ 0, %entry ]84 %l2 = getelementptr inbounds i32, ptr %A, i32 %indvars.iv85 %l3 = load i32, ptr %l2, align 486 %l4 = getelementptr inbounds i32, ptr %B, i32 %indvars.iv87 %l5 = load i32, ptr %l4, align 488 %l7 = add i32 %sum.02, %indvars.iv89 %l8 = add i32 %l7, %l390 %l9 = add i32 %l8, %l591 %indvars.iv.next = add i32 %indvars.iv, 192 %exitcond = icmp eq i32 %indvars.iv.next, 25793 br i1 %exitcond, label %._crit_edge, label %.lr.ph94 95._crit_edge: ; preds = %.lr.ph96 %sum.0.lcssa = phi i32 [ %l9, %.lr.ph ]97 ret i32 %sum.0.lcssa98}99 100define i32 @reduction_prod(ptr noalias nocapture %A, ptr noalias nocapture %B) {101; CHECK-LABEL: @reduction_prod(102; CHECK-NEXT: entry:103; CHECK-NEXT: br label [[VECTOR_PH:%.*]]104; CHECK: vector.ph:105; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]106; CHECK: vector.body:107; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]108; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 1), [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]109; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 257)110; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[INDEX]]111; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)112; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i32 [[INDEX]]113; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)114; CHECK-NEXT: [[TMP2:%.*]] = mul <4 x i32> [[VEC_PHI]], [[WIDE_MASKED_LOAD]]115; CHECK-NEXT: [[TMP3:%.*]] = mul <4 x i32> [[TMP2]], [[WIDE_MASKED_LOAD1]]116; CHECK-NEXT: [[TMP4]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP3]], <4 x i32> [[VEC_PHI]]117; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4118; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260119; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]120; CHECK: middle.block:121; CHECK-NEXT: br label [[DOTLR_PH:%.*]]122; CHECK: ._crit_edge:123; CHECK-NEXT: [[PROD_0_LCSSA:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP4]])124; CHECK-NEXT: ret i32 [[PROD_0_LCSSA]]125;126entry:127 br label %.lr.ph128 129.lr.ph: ; preds = %entry, %.lr.ph130 %indvars.iv = phi i32 [ %indvars.iv.next, %.lr.ph ], [ 0, %entry ]131 %prod.02 = phi i32 [ %l9, %.lr.ph ], [ 1, %entry ]132 %l2 = getelementptr inbounds i32, ptr %A, i32 %indvars.iv133 %l3 = load i32, ptr %l2, align 4134 %l4 = getelementptr inbounds i32, ptr %B, i32 %indvars.iv135 %l5 = load i32, ptr %l4, align 4136 %l8 = mul i32 %prod.02, %l3137 %l9 = mul i32 %l8, %l5138 %indvars.iv.next = add i32 %indvars.iv, 1139 %exitcond = icmp eq i32 %indvars.iv.next, 257140 br i1 %exitcond, label %._crit_edge, label %.lr.ph141 142._crit_edge: ; preds = %.lr.ph143 %prod.0.lcssa = phi i32 [ %l9, %.lr.ph ]144 ret i32 %prod.0.lcssa145}146 147define i32 @reduction_and(ptr nocapture %A, ptr nocapture %B) {148; CHECK-LABEL: @reduction_and(149; CHECK-NEXT: entry:150; CHECK-NEXT: br label [[VECTOR_PH:%.*]]151; CHECK: vector.ph:152; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]153; CHECK: vector.body:154; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]155; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]156; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 257)157; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[INDEX]]158; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)159; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i32 [[INDEX]]160; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)161; CHECK-NEXT: [[TMP2:%.*]] = and <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]]162; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP2]], <4 x i32> splat (i32 -1)163; CHECK-NEXT: [[TMP4]] = and <4 x i32> [[VEC_PHI]], [[TMP3]]164; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4165; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260166; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]167; CHECK: middle.block:168; CHECK-NEXT: br label [[FOR_BODY:%.*]]169; CHECK: for.end:170; CHECK-NEXT: [[RESULT_0_LCSSA:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP4]])171; CHECK-NEXT: ret i32 [[RESULT_0_LCSSA]]172;173entry:174 br label %for.body175 176for.body: ; preds = %entry, %for.body177 %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %entry ]178 %result.08 = phi i32 [ %and, %for.body ], [ -1, %entry ]179 %arrayidx = getelementptr inbounds i32, ptr %A, i32 %indvars.iv180 %l0 = load i32, ptr %arrayidx, align 4181 %arrayidx2 = getelementptr inbounds i32, ptr %B, i32 %indvars.iv182 %l1 = load i32, ptr %arrayidx2, align 4183 %add = and i32 %result.08, %l0184 %and = and i32 %add, %l1185 %indvars.iv.next = add i32 %indvars.iv, 1186 %exitcond = icmp eq i32 %indvars.iv.next, 257187 br i1 %exitcond, label %for.end, label %for.body188 189for.end: ; preds = %for.body, %entry190 %result.0.lcssa = phi i32 [ %and, %for.body ]191 ret i32 %result.0.lcssa192}193 194define i32 @reduction_or(ptr nocapture %A, ptr nocapture %B) {195; CHECK-LABEL: @reduction_or(196; CHECK-NEXT: entry:197; CHECK-NEXT: br label [[VECTOR_PH:%.*]]198; CHECK: vector.ph:199; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]200; CHECK: vector.body:201; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]202; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]203; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 257)204; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[INDEX]]205; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)206; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i32 [[INDEX]]207; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)208; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD1]], [[WIDE_MASKED_LOAD]]209; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP2]], <4 x i32> zeroinitializer210; CHECK-NEXT: [[TMP4]] = or <4 x i32> [[VEC_PHI]], [[TMP3]]211; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4212; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260213; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]214; CHECK: middle.block:215; CHECK-NEXT: br label [[FOR_BODY:%.*]]216; CHECK: for.end:217; CHECK-NEXT: [[RESULT_0_LCSSA:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP4]])218; CHECK-NEXT: ret i32 [[RESULT_0_LCSSA]]219;220entry:221 br label %for.body222 223for.body: ; preds = %entry, %for.body224 %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %entry ]225 %result.08 = phi i32 [ %or, %for.body ], [ 0, %entry ]226 %arrayidx = getelementptr inbounds i32, ptr %A, i32 %indvars.iv227 %l0 = load i32, ptr %arrayidx, align 4228 %arrayidx2 = getelementptr inbounds i32, ptr %B, i32 %indvars.iv229 %l1 = load i32, ptr %arrayidx2, align 4230 %add = add nsw i32 %l1, %l0231 %or = or i32 %add, %result.08232 %indvars.iv.next = add i32 %indvars.iv, 1233 %exitcond = icmp eq i32 %indvars.iv.next, 257234 br i1 %exitcond, label %for.end, label %for.body235 236for.end: ; preds = %for.body, %entry237 %result.0.lcssa = phi i32 [ %or, %for.body ]238 ret i32 %result.0.lcssa239}240 241define i32 @reduction_xor(ptr nocapture %A, ptr nocapture %B) {242; CHECK-LABEL: @reduction_xor(243; CHECK-NEXT: entry:244; CHECK-NEXT: br label [[VECTOR_PH:%.*]]245; CHECK: vector.ph:246; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]247; CHECK: vector.body:248; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]249; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]250; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 257)251; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[INDEX]]252; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)253; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i32 [[INDEX]]254; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP1]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)255; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD1]], [[WIDE_MASKED_LOAD]]256; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> [[TMP2]], <4 x i32> zeroinitializer257; CHECK-NEXT: [[TMP4]] = xor <4 x i32> [[VEC_PHI]], [[TMP3]]258; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4259; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260260; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]261; CHECK: middle.block:262; CHECK-NEXT: br label [[FOR_BODY:%.*]]263; CHECK: for.end:264; CHECK-NEXT: [[RESULT_0_LCSSA:%.*]] = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> [[TMP4]])265; CHECK-NEXT: ret i32 [[RESULT_0_LCSSA]]266;267entry:268 br label %for.body269 270for.body: ; preds = %entry, %for.body271 %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %entry ]272 %result.08 = phi i32 [ %xor, %for.body ], [ 0, %entry ]273 %arrayidx = getelementptr inbounds i32, ptr %A, i32 %indvars.iv274 %l0 = load i32, ptr %arrayidx, align 4275 %arrayidx2 = getelementptr inbounds i32, ptr %B, i32 %indvars.iv276 %l1 = load i32, ptr %arrayidx2, align 4277 %add = add nsw i32 %l1, %l0278 %xor = xor i32 %add, %result.08279 %indvars.iv.next = add i32 %indvars.iv, 1280 %exitcond = icmp eq i32 %indvars.iv.next, 257281 br i1 %exitcond, label %for.end, label %for.body282 283for.end: ; preds = %for.body, %entry284 %result.0.lcssa = phi i32 [ %xor, %for.body ]285 ret i32 %result.0.lcssa286}287 288define float @reduction_fadd(ptr nocapture %A, ptr nocapture %B) {289; CHECK-LABEL: @reduction_fadd(290; CHECK-NEXT: entry:291; CHECK-NEXT: br label [[VECTOR_PH:%.*]]292; CHECK: vector.ph:293; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]294; CHECK: vector.body:295; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]296; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]297; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 257)298; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i32 [[INDEX]]299; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x float> poison)300; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i32 [[INDEX]]301; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 4 [[TMP1]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x float> poison)302; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[WIDE_MASKED_LOAD]]303; CHECK-NEXT: [[TMP3:%.*]] = fadd fast <4 x float> [[TMP2]], [[WIDE_MASKED_LOAD1]]304; CHECK-NEXT: [[TMP4]] = select fast <4 x i1> [[ACTIVE_LANE_MASK]], <4 x float> [[TMP3]], <4 x float> [[VEC_PHI]]305; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4306; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260307; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]308; CHECK: middle.block:309; CHECK-NEXT: br label [[FOR_BODY:%.*]]310; CHECK: for.end:311; CHECK-NEXT: [[RESULT_0_LCSSA:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP4]])312; CHECK-NEXT: ret float [[RESULT_0_LCSSA]]313;314entry:315 br label %for.body316 317for.body: ; preds = %entry, %for.body318 %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %entry ]319 %result.08 = phi float [ %fadd, %for.body ], [ 0.0, %entry ]320 %arrayidx = getelementptr inbounds float, ptr %A, i32 %indvars.iv321 %l0 = load float, ptr %arrayidx, align 4322 %arrayidx2 = getelementptr inbounds float, ptr %B, i32 %indvars.iv323 %l1 = load float, ptr %arrayidx2, align 4324 %add = fadd fast float %result.08, %l0325 %fadd = fadd fast float %add, %l1326 %indvars.iv.next = add i32 %indvars.iv, 1327 %exitcond = icmp eq i32 %indvars.iv.next, 257328 br i1 %exitcond, label %for.end, label %for.body329 330for.end: ; preds = %for.body, %entry331 %result.0.lcssa = phi float [ %fadd, %for.body ]332 ret float %result.0.lcssa333}334 335define float @reduction_fmul(ptr nocapture %A, ptr nocapture %B) {336; CHECK-LABEL: @reduction_fmul(337; CHECK-NEXT: entry:338; CHECK-NEXT: br label [[VECTOR_PH:%.*]]339; CHECK: vector.ph:340; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]341; CHECK: vector.body:342; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]343; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float 0.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]344; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 257)345; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i32 [[INDEX]]346; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x float> poison)347; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i32 [[INDEX]]348; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 4 [[TMP1]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x float> poison)349; CHECK-NEXT: [[TMP2:%.*]] = fmul fast <4 x float> [[VEC_PHI]], [[WIDE_MASKED_LOAD]]350; CHECK-NEXT: [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[WIDE_MASKED_LOAD1]]351; CHECK-NEXT: [[TMP4]] = select fast <4 x i1> [[ACTIVE_LANE_MASK]], <4 x float> [[TMP3]], <4 x float> [[VEC_PHI]]352; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4353; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 260354; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]355; CHECK: middle.block:356; CHECK-NEXT: br label [[FOR_BODY:%.*]]357; CHECK: for.end:358; CHECK-NEXT: [[RESULT_0_LCSSA:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[TMP4]])359; CHECK-NEXT: ret float [[RESULT_0_LCSSA]]360;361entry:362 br label %for.body363 364for.body: ; preds = %entry, %for.body365 %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %entry ]366 %result.08 = phi float [ %fmul, %for.body ], [ 0.0, %entry ]367 %arrayidx = getelementptr inbounds float, ptr %A, i32 %indvars.iv368 %l0 = load float, ptr %arrayidx, align 4369 %arrayidx2 = getelementptr inbounds float, ptr %B, i32 %indvars.iv370 %l1 = load float, ptr %arrayidx2, align 4371 %add = fmul fast float %result.08, %l0372 %fmul = fmul fast float %add, %l1373 %indvars.iv.next = add i32 %indvars.iv, 1374 %exitcond = icmp eq i32 %indvars.iv.next, 257375 br i1 %exitcond, label %for.end, label %for.body376 377for.end: ; preds = %for.body, %entry378 %result.0.lcssa = phi float [ %fmul, %for.body ]379 ret float %result.0.lcssa380}381 382define i32 @reduction_min(ptr nocapture %A, ptr nocapture %B) {383; CHECK-LABEL: @reduction_min(384; CHECK-NEXT: entry:385; CHECK-NEXT: br label [[VECTOR_PH:%.*]]386; CHECK: vector.ph:387; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]388; CHECK: vector.body:389; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]390; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 1000), [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]391; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[INDEX]]392; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4393; CHECK-NEXT: [[TMP1]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])394; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4395; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256396; CHECK-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]397; CHECK: middle.block:398; CHECK-NEXT: br label [[SCALAR_PH:%.*]]399; CHECK: scalar.ph:400; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> [[TMP1]])401; CHECK-NEXT: br label [[FOR_BODY:%.*]]402; CHECK: for.body:403; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 256, [[SCALAR_PH]] ]404; CHECK-NEXT: [[RESULT_08:%.*]] = phi i32 [ [[V0:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]405; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDVARS_IV]]406; CHECK-NEXT: [[L0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4407; CHECK-NEXT: [[V0]] = call i32 @llvm.smin.i32(i32 [[RESULT_08]], i32 [[L0]])408; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i32 [[INDVARS_IV]], 1409; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], 257410; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]411; CHECK: for.end:412; CHECK-NEXT: ret i32 [[V0]]413;414entry:415 br label %for.body416 417for.body: ; preds = %entry, %for.body418 %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %entry ]419 %result.08 = phi i32 [ %v0, %for.body ], [ 1000, %entry ]420 %arrayidx = getelementptr inbounds i32, ptr %A, i32 %indvars.iv421 %l0 = load i32, ptr %arrayidx, align 4422 %c0 = icmp slt i32 %result.08, %l0423 %v0 = select i1 %c0, i32 %result.08, i32 %l0424 %indvars.iv.next = add i32 %indvars.iv, 1425 %exitcond = icmp eq i32 %indvars.iv.next, 257426 br i1 %exitcond, label %for.end, label %for.body427 428for.end: ; preds = %for.body, %entry429 %result.0.lcssa = phi i32 [ %v0, %for.body ]430 ret i32 %result.0.lcssa431}432 433define i32 @reduction_max(ptr nocapture %A, ptr nocapture %B) {434; CHECK-LABEL: @reduction_max(435; CHECK-NEXT: entry:436; CHECK-NEXT: br label [[VECTOR_PH:%.*]]437; CHECK: vector.ph:438; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]439; CHECK: vector.body:440; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]441; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 1000), [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]442; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[INDEX]]443; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4444; CHECK-NEXT: [[TMP1]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> [[VEC_PHI]], <4 x i32> [[WIDE_LOAD]])445; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4446; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256447; CHECK-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]448; CHECK: middle.block:449; CHECK-NEXT: br label [[SCALAR_PH:%.*]]450; CHECK: scalar.ph:451; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> [[TMP1]])452; CHECK-NEXT: br label [[FOR_BODY:%.*]]453; CHECK: for.body:454; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 256, [[SCALAR_PH]] ]455; CHECK-NEXT: [[RESULT_08:%.*]] = phi i32 [ [[V0:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]456; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDVARS_IV]]457; CHECK-NEXT: [[L0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4458; CHECK-NEXT: [[V0]] = call i32 @llvm.umax.i32(i32 [[RESULT_08]], i32 [[L0]])459; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i32 [[INDVARS_IV]], 1460; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], 257461; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]462; CHECK: for.end:463; CHECK-NEXT: ret i32 [[V0]]464;465entry:466 br label %for.body467 468for.body: ; preds = %entry, %for.body469 %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %entry ]470 %result.08 = phi i32 [ %v0, %for.body ], [ 1000, %entry ]471 %arrayidx = getelementptr inbounds i32, ptr %A, i32 %indvars.iv472 %l0 = load i32, ptr %arrayidx, align 4473 %c0 = icmp ugt i32 %result.08, %l0474 %v0 = select i1 %c0, i32 %result.08, i32 %l0475 %indvars.iv.next = add i32 %indvars.iv, 1476 %exitcond = icmp eq i32 %indvars.iv.next, 257477 br i1 %exitcond, label %for.end, label %for.body478 479for.end: ; preds = %for.body, %entry480 %result.0.lcssa = phi i32 [ %v0, %for.body ]481 ret i32 %result.0.lcssa482}483 484define float @reduction_fmax(ptr nocapture %A, ptr nocapture %B) {485; CHECK-LABEL: @reduction_fmax(486; CHECK-NEXT: entry:487; CHECK-NEXT: br label [[FOR_BODY:%.*]]488; CHECK: for.body:489; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]490; CHECK-NEXT: [[RESULT_08:%.*]] = phi float [ [[V0:%.*]], [[FOR_BODY]] ], [ 1.000000e+03, [[ENTRY]] ]491; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i32 [[INDVARS_IV]]492; CHECK-NEXT: [[L0:%.*]] = load float, ptr [[ARRAYIDX]], align 4493; CHECK-NEXT: [[C0:%.*]] = fcmp ogt float [[RESULT_08]], [[L0]]494; CHECK-NEXT: [[V0]] = select i1 [[C0]], float [[RESULT_08]], float [[L0]]495; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i32 [[INDVARS_IV]], 1496; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], 257497; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]498; CHECK: for.end:499; CHECK-NEXT: ret float [[V0]]500;501entry:502 br label %for.body503 504for.body: ; preds = %entry, %for.body505 %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %entry ]506 %result.08 = phi float [ %v0, %for.body ], [ 1000.0, %entry ]507 %arrayidx = getelementptr inbounds float, ptr %A, i32 %indvars.iv508 %l0 = load float, ptr %arrayidx, align 4509 %c0 = fcmp ogt float %result.08, %l0510 %v0 = select i1 %c0, float %result.08, float %l0511 %indvars.iv.next = add i32 %indvars.iv, 1512 %exitcond = icmp eq i32 %indvars.iv.next, 257513 br i1 %exitcond, label %for.end, label %for.body514 515for.end: ; preds = %for.body, %entry516 %result.0.lcssa = phi float [ %v0, %for.body ]517 ret float %result.0.lcssa518}519