1261 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 52; RUN: opt < %s -p loop-vectorize -mtriple riscv64 -mattr=+v -S | FileCheck %s3 4; Reduction can be vectorized5 6; ADD7 8define i32 @add(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {9; CHECK-LABEL: define i32 @add(10; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {11; CHECK-NEXT: [[ENTRY:.*:]]12; CHECK-NEXT: br label %[[VECTOR_PH:.*]]13; CHECK: [[VECTOR_PH]]:14; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]15; CHECK: [[VECTOR_BODY]]:16; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]17; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ insertelement (<vscale x 4 x i32> zeroinitializer, i32 2, i32 0), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]18; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]19; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)20; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]21; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP13]])22; CHECK-NEXT: [[TMP7:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]23; CHECK-NEXT: [[TMP8]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP13]])24; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP13]] to i6425; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]]26; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]27; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 028; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]29; CHECK: [[MIDDLE_BLOCK]]:30; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP8]])31; CHECK-NEXT: br label %[[FOR_END:.*]]32; CHECK: [[FOR_END]]:33; CHECK-NEXT: ret i32 [[TMP11]]34;35entry:36 br label %for.body37 38for.body: ; preds = %entry, %for.body39 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]40 %sum.07 = phi i32 [ 2, %entry ], [ %add, %for.body ]41 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv42 %0 = load i32, ptr %arrayidx, align 443 %add = add nsw i32 %0, %sum.0744 %iv.next = add nuw nsw i64 %iv, 145 %exitcond.not = icmp eq i64 %iv.next, %n46 br i1 %exitcond.not, label %for.end, label %for.body47 48for.end: ; preds = %for.body, %entry49 ret i32 %add50}51 52define i32 @sub(ptr %a, i64 %n) {53; CHECK-LABEL: define i32 @sub(54; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {55; CHECK-NEXT: [[ENTRY:.*:]]56; CHECK-NEXT: br label %[[VECTOR_PH:.*]]57; CHECK: [[VECTOR_PH]]:58; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]59; CHECK: [[VECTOR_BODY]]:60; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]61; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ insertelement (<vscale x 4 x i32> zeroinitializer, i32 1024, i32 0), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]62; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]63; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)64; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]65; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP0]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])66; CHECK-NEXT: [[TMP2:%.*]] = sub <vscale x 4 x i32> [[VEC_PHI]], [[VP_OP_LOAD]]67; CHECK-NEXT: [[TMP3]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP1]])68; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP1]] to i6469; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP4]], [[INDEX]]70; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]71; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 072; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]73; CHECK: [[MIDDLE_BLOCK]]:74; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP3]])75; CHECK-NEXT: br label %[[EXIT:.*]]76; CHECK: [[EXIT]]:77; CHECK-NEXT: ret i32 [[TMP6]]78;79entry:80 br label %loop81 82loop:83 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]84 %rdx = phi i32 [ 1024, %entry ], [ %sub, %loop ]85 %gep = getelementptr i32, ptr %a, i64 %iv86 %x = load i32, ptr %gep87 %sub = sub i32 %rdx, %x88 %iv.next = add i64 %iv, 189 %done = icmp eq i64 %iv.next, %n90 br i1 %done, label %exit, label %loop91 92exit:93 ret i32 %sub94}95 96define i32 @addsub(ptr %a, ptr %b, i64 %n) {97; CHECK-LABEL: define i32 @addsub(98; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {99; CHECK-NEXT: [[ENTRY:.*:]]100; CHECK-NEXT: br label %[[VECTOR_PH:.*]]101; CHECK: [[VECTOR_PH]]:102; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]103; CHECK: [[VECTOR_BODY]]:104; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]105; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]106; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]107; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)108; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]109; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP0]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])110; CHECK-NEXT: [[TMP2:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[VP_OP_LOAD]]111; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]112; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])113; CHECK-NEXT: [[TMP9:%.*]] = sub <vscale x 4 x i32> [[TMP2]], [[VP_OP_LOAD1]]114; CHECK-NEXT: [[TMP5]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP9]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP1]])115; CHECK-NEXT: [[TMP6:%.*]] = zext i32 [[TMP1]] to i64116; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP6]], [[INDEX]]117; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP6]]118; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 0119; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]120; CHECK: [[MIDDLE_BLOCK]]:121; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP5]])122; CHECK-NEXT: br label %[[EXIT:.*]]123; CHECK: [[EXIT]]:124; CHECK-NEXT: ret i32 [[TMP8]]125;126entry:127 br label %loop128 129loop:130 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]131 %rdx = phi i32 [ 0, %entry ], [ %sub, %loop ]132 %gep.a = getelementptr i32, ptr %a, i64 %iv133 %x = load i32, ptr %gep.a134 %add = add i32 %rdx, %x135 %gep.b = getelementptr i32, ptr %b, i64 %iv136 %y = load i32, ptr %gep.b137 %sub = sub i32 %add, %y138 %iv.next = add i64 %iv, 1139 %done = icmp eq i64 %iv.next, %n140 br i1 %done, label %exit, label %loop141 142exit:143 ret i32 %sub144}145 146 147; OR148 149define i32 @or(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {150; CHECK-LABEL: define i32 @or(151; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {152; CHECK-NEXT: [[ENTRY:.*:]]153; CHECK-NEXT: br label %[[VECTOR_PH:.*]]154; CHECK: [[VECTOR_PH]]:155; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]156; CHECK: [[VECTOR_BODY]]:157; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]158; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ insertelement (<vscale x 4 x i32> zeroinitializer, i32 2, i32 0), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]159; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]160; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)161; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]162; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP13]])163; CHECK-NEXT: [[TMP7:%.*]] = or <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]164; CHECK-NEXT: [[TMP8]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP13]])165; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP13]] to i64166; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]]167; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]168; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0169; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]170; CHECK: [[MIDDLE_BLOCK]]:171; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[TMP8]])172; CHECK-NEXT: br label %[[FOR_END:.*]]173; CHECK: [[FOR_END]]:174; CHECK-NEXT: ret i32 [[TMP11]]175;176entry:177 br label %for.body178 179for.body: ; preds = %entry, %for.body180 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]181 %sum.07 = phi i32 [ 2, %entry ], [ %or, %for.body ]182 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv183 %0 = load i32, ptr %arrayidx, align 4184 %or = or i32 %0, %sum.07185 %iv.next = add nuw nsw i64 %iv, 1186 %exitcond.not = icmp eq i64 %iv.next, %n187 br i1 %exitcond.not, label %for.end, label %for.body188 189for.end: ; preds = %for.body, %entry190 ret i32 %or191}192 193; AND194 195define i32 @and(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {196; CHECK-LABEL: define i32 @and(197; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {198; CHECK-NEXT: [[ENTRY:.*:]]199; CHECK-NEXT: br label %[[VECTOR_PH:.*]]200; CHECK: [[VECTOR_PH]]:201; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]202; CHECK: [[VECTOR_BODY]]:203; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]204; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ insertelement (<vscale x 4 x i32> splat (i32 -1), i32 2, i32 0), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]205; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]206; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)207; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]208; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP13]])209; CHECK-NEXT: [[TMP7:%.*]] = and <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]210; CHECK-NEXT: [[TMP8]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP13]])211; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP13]] to i64212; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]]213; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]214; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0215; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]216; CHECK: [[MIDDLE_BLOCK]]:217; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> [[TMP8]])218; CHECK-NEXT: br label %[[FOR_END:.*]]219; CHECK: [[FOR_END]]:220; CHECK-NEXT: ret i32 [[TMP11]]221;222entry:223 br label %for.body224 225for.body: ; preds = %entry, %for.body226 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]227 %sum.07 = phi i32 [ 2, %entry ], [ %and, %for.body ]228 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv229 %0 = load i32, ptr %arrayidx, align 4230 %and = and i32 %0, %sum.07231 %iv.next = add nuw nsw i64 %iv, 1232 %exitcond.not = icmp eq i64 %iv.next, %n233 br i1 %exitcond.not, label %for.end, label %for.body234 235for.end: ; preds = %for.body, %entry236 ret i32 %and237}238 239; XOR240 241define i32 @xor(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {242; CHECK-LABEL: define i32 @xor(243; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {244; CHECK-NEXT: [[ENTRY:.*:]]245; CHECK-NEXT: br label %[[VECTOR_PH:.*]]246; CHECK: [[VECTOR_PH]]:247; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]248; CHECK: [[VECTOR_BODY]]:249; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]250; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ insertelement (<vscale x 4 x i32> zeroinitializer, i32 2, i32 0), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]251; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]252; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)253; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]254; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP13]])255; CHECK-NEXT: [[TMP7:%.*]] = xor <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]256; CHECK-NEXT: [[TMP8]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP13]])257; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP13]] to i64258; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]]259; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]260; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0261; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]262; CHECK: [[MIDDLE_BLOCK]]:263; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> [[TMP8]])264; CHECK-NEXT: br label %[[FOR_END:.*]]265; CHECK: [[FOR_END]]:266; CHECK-NEXT: ret i32 [[TMP11]]267;268entry:269 br label %for.body270 271for.body: ; preds = %entry, %for.body272 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]273 %sum.07 = phi i32 [ 2, %entry ], [ %xor, %for.body ]274 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv275 %0 = load i32, ptr %arrayidx, align 4276 %xor = xor i32 %0, %sum.07277 %iv.next = add nuw nsw i64 %iv, 1278 %exitcond.not = icmp eq i64 %iv.next, %n279 br i1 %exitcond.not, label %for.end, label %for.body280 281for.end: ; preds = %for.body, %entry282 ret i32 %xor283}284 285; SMIN286 287define i32 @smin(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {288; CHECK-LABEL: define i32 @smin(289; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {290; CHECK-NEXT: [[ENTRY:.*:]]291; CHECK-NEXT: br label %[[VECTOR_PH:.*]]292; CHECK: [[VECTOR_PH]]:293; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]294; CHECK: [[VECTOR_BODY]]:295; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]296; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ splat (i32 2), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]297; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]298; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)299; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]300; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])301; CHECK-NEXT: [[TMP7:%.*]] = icmp slt <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]302; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]303; CHECK-NEXT: [[TMP9]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP8]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP14]])304; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP14]] to i64305; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]306; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]307; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0308; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]309; CHECK: [[MIDDLE_BLOCK]]:310; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[TMP9]])311; CHECK-NEXT: br label %[[FOR_END:.*]]312; CHECK: [[FOR_END]]:313; CHECK-NEXT: ret i32 [[TMP12]]314;315entry:316 br label %for.body317 318for.body: ; preds = %entry, %for.body319 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]320 %sum.010 = phi i32 [ 2, %entry ], [ %.sroa.speculated, %for.body ]321 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv322 %0 = load i32, ptr %arrayidx, align 4323 %cmp.i = icmp slt i32 %0, %sum.010324 %.sroa.speculated = select i1 %cmp.i, i32 %0, i32 %sum.010325 %iv.next = add nuw nsw i64 %iv, 1326 %exitcond.not = icmp eq i64 %iv.next, %n327 br i1 %exitcond.not, label %for.end, label %for.body328 329for.end:330 ret i32 %.sroa.speculated331}332 333; UMAX334 335define i32 @umax(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {336; CHECK-LABEL: define i32 @umax(337; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {338; CHECK-NEXT: [[ENTRY:.*:]]339; CHECK-NEXT: br label %[[VECTOR_PH:.*]]340; CHECK: [[VECTOR_PH]]:341; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]342; CHECK: [[VECTOR_BODY]]:343; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]344; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ splat (i32 2), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]345; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]346; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)347; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]348; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])349; CHECK-NEXT: [[TMP7:%.*]] = icmp ugt <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]350; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]351; CHECK-NEXT: [[TMP9]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP8]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP14]])352; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP14]] to i64353; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]354; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]355; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0356; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]357; CHECK: [[MIDDLE_BLOCK]]:358; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> [[TMP9]])359; CHECK-NEXT: br label %[[FOR_END:.*]]360; CHECK: [[FOR_END]]:361; CHECK-NEXT: ret i32 [[TMP12]]362;363entry:364 br label %for.body365 366for.body: ; preds = %entry, %for.body367 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]368 %sum.010 = phi i32 [ 2, %entry ], [ %.sroa.speculated, %for.body ]369 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv370 %0 = load i32, ptr %arrayidx, align 4371 %cmp.i = icmp ugt i32 %0, %sum.010372 %.sroa.speculated = select i1 %cmp.i, i32 %0, i32 %sum.010373 %iv.next = add nuw nsw i64 %iv, 1374 %exitcond.not = icmp eq i64 %iv.next, %n375 br i1 %exitcond.not, label %for.end, label %for.body376 377for.end:378 ret i32 %.sroa.speculated379}380 381; FADD (FAST)382 383define float @fadd_fast(ptr noalias nocapture readonly %a, i64 %n) {384; CHECK-LABEL: define float @fadd_fast(385; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {386; CHECK-NEXT: [[ENTRY:.*:]]387; CHECK-NEXT: br label %[[VECTOR_PH:.*]]388; CHECK: [[VECTOR_PH]]:389; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]390; CHECK: [[VECTOR_BODY]]:391; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]392; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]393; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]394; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)395; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]396; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP13]])397; CHECK-NEXT: [[TMP7:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]398; CHECK-NEXT: [[TMP8]] = call <vscale x 4 x float> @llvm.vp.merge.nxv4f32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> [[TMP7]], <vscale x 4 x float> [[VEC_PHI]], i32 [[TMP13]])399; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP13]] to i64400; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]]401; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]402; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0403; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]404; CHECK: [[MIDDLE_BLOCK]]:405; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP8]])406; CHECK-NEXT: br label %[[FOR_END:.*]]407; CHECK: [[FOR_END]]:408; CHECK-NEXT: ret float [[TMP11]]409;410entry:411 br label %for.body412 413for.body:414 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]415 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]416 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv417 %0 = load float, ptr %arrayidx, align 4418 %add = fadd fast float %0, %sum.07419 %iv.next = add nuw nsw i64 %iv, 1420 %exitcond.not = icmp eq i64 %iv.next, %n421 br i1 %exitcond.not, label %for.end, label %for.body422 423for.end:424 ret float %add425}426 427define half @fadd_fast_half_zvfh(ptr noalias nocapture readonly %a, i64 %n) "target-features"="+zvfh" {428; CHECK-LABEL: define half @fadd_fast_half_zvfh(429; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {430; CHECK-NEXT: [[ENTRY:.*:]]431; CHECK-NEXT: br label %[[VECTOR_PH:.*]]432; CHECK: [[VECTOR_PH]]:433; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]434; CHECK: [[VECTOR_BODY]]:435; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]436; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 8 x half> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]437; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]438; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)439; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds half, ptr [[A]], i64 [[INDEX]]440; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 4 [[TMP6]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])441; CHECK-NEXT: [[TMP7:%.*]] = fadd fast <vscale x 8 x half> [[WIDE_LOAD]], [[VEC_PHI]]442; CHECK-NEXT: [[TMP8]] = call <vscale x 8 x half> @llvm.vp.merge.nxv8f16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x half> [[TMP7]], <vscale x 8 x half> [[VEC_PHI]], i32 [[TMP13]])443; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP13]] to i64444; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]]445; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]446; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0447; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]448; CHECK: [[MIDDLE_BLOCK]]:449; CHECK-NEXT: [[TMP11:%.*]] = call fast half @llvm.vector.reduce.fadd.nxv8f16(half 0xH0000, <vscale x 8 x half> [[TMP8]])450; CHECK-NEXT: br label %[[FOR_END:.*]]451; CHECK: [[FOR_END]]:452; CHECK-NEXT: ret half [[TMP11]]453;454entry:455 br label %for.body456 457for.body:458 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]459 %sum.07 = phi half [ 0.000000e+00, %entry ], [ %add, %for.body ]460 %arrayidx = getelementptr inbounds half, ptr %a, i64 %iv461 %0 = load half, ptr %arrayidx, align 4462 %add = fadd fast half %0, %sum.07463 %iv.next = add nuw nsw i64 %iv, 1464 %exitcond.not = icmp eq i64 %iv.next, %n465 br i1 %exitcond.not, label %for.end, label %for.body466 467for.end:468 ret half %add469}470 471define half @fadd_fast_half_zvfhmin(ptr noalias nocapture readonly %a, i64 %n) "target-features"="+zvfhmin" {472; CHECK-LABEL: define half @fadd_fast_half_zvfhmin(473; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR2:[0-9]+]] {474; CHECK-NEXT: [[ENTRY:.*]]:475; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 32476; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]477; CHECK: [[VECTOR_PH]]:478; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 32479; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]480; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]481; CHECK: [[VECTOR_BODY]]:482; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]483; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x half> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]484; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <16 x half> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]485; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds half, ptr [[A]], i64 [[INDEX]]486; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds half, ptr [[TMP0]], i64 16487; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x half>, ptr [[TMP0]], align 4488; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x half>, ptr [[TMP1]], align 4489; CHECK-NEXT: [[TMP2]] = fadd fast <16 x half> [[WIDE_LOAD]], [[VEC_PHI]]490; CHECK-NEXT: [[TMP3]] = fadd fast <16 x half> [[WIDE_LOAD2]], [[VEC_PHI1]]491; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32492; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]493; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]494; CHECK: [[MIDDLE_BLOCK]]:495; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd fast <16 x half> [[TMP3]], [[TMP2]]496; CHECK-NEXT: [[TMP5:%.*]] = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> [[BIN_RDX]])497; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]498; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]499; CHECK: [[SCALAR_PH]]:500; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]501; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi half [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ 0xH0000, %[[ENTRY]] ]502; CHECK-NEXT: br label %[[FOR_BODY:.*]]503; CHECK: [[FOR_BODY]]:504; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]505; CHECK-NEXT: [[SUM_07:%.*]] = phi half [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]506; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds half, ptr [[A]], i64 [[IV]]507; CHECK-NEXT: [[TMP6:%.*]] = load half, ptr [[ARRAYIDX]], align 4508; CHECK-NEXT: [[ADD]] = fadd fast half [[TMP6]], [[SUM_07]]509; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1510; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]511; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]512; CHECK: [[FOR_END]]:513; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi half [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ]514; CHECK-NEXT: ret half [[ADD_LCSSA]]515;516entry:517 br label %for.body518 519for.body:520 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]521 %sum.07 = phi half [ 0.000000e+00, %entry ], [ %add, %for.body ]522 %arrayidx = getelementptr inbounds half, ptr %a, i64 %iv523 %0 = load half, ptr %arrayidx, align 4524 %add = fadd fast half %0, %sum.07525 %iv.next = add nuw nsw i64 %iv, 1526 %exitcond.not = icmp eq i64 %iv.next, %n527 br i1 %exitcond.not, label %for.end, label %for.body528 529for.end:530 ret half %add531}532 533define bfloat @fadd_fast_bfloat(ptr noalias nocapture readonly %a, i64 %n) "target-features"="+zvfbfmin" {534; CHECK-LABEL: define bfloat @fadd_fast_bfloat(535; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR3:[0-9]+]] {536; CHECK-NEXT: [[ENTRY:.*]]:537; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 32538; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]539; CHECK: [[VECTOR_PH]]:540; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 32541; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]542; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]543; CHECK: [[VECTOR_BODY]]:544; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]545; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x bfloat> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]546; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <16 x bfloat> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]547; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds bfloat, ptr [[A]], i64 [[INDEX]]548; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds bfloat, ptr [[TMP0]], i64 16549; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x bfloat>, ptr [[TMP0]], align 4550; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x bfloat>, ptr [[TMP1]], align 4551; CHECK-NEXT: [[TMP2]] = fadd fast <16 x bfloat> [[WIDE_LOAD]], [[VEC_PHI]]552; CHECK-NEXT: [[TMP3]] = fadd fast <16 x bfloat> [[WIDE_LOAD2]], [[VEC_PHI1]]553; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32554; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]555; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]556; CHECK: [[MIDDLE_BLOCK]]:557; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd fast <16 x bfloat> [[TMP3]], [[TMP2]]558; CHECK-NEXT: [[TMP5:%.*]] = call fast bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0xR0000, <16 x bfloat> [[BIN_RDX]])559; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]560; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]561; CHECK: [[SCALAR_PH]]:562; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]563; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi bfloat [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ 0xR0000, %[[ENTRY]] ]564; CHECK-NEXT: br label %[[FOR_BODY:.*]]565; CHECK: [[FOR_BODY]]:566; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]567; CHECK-NEXT: [[SUM_07:%.*]] = phi bfloat [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]568; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds bfloat, ptr [[A]], i64 [[IV]]569; CHECK-NEXT: [[TMP6:%.*]] = load bfloat, ptr [[ARRAYIDX]], align 4570; CHECK-NEXT: [[ADD]] = fadd fast bfloat [[TMP6]], [[SUM_07]]571; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1572; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]573; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]574; CHECK: [[FOR_END]]:575; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi bfloat [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ]576; CHECK-NEXT: ret bfloat [[ADD_LCSSA]]577;578entry:579 br label %for.body580 581for.body:582 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]583 %sum.07 = phi bfloat [ 0.000000e+00, %entry ], [ %add, %for.body ]584 %arrayidx = getelementptr inbounds bfloat, ptr %a, i64 %iv585 %0 = load bfloat, ptr %arrayidx, align 4586 %add = fadd fast bfloat %0, %sum.07587 %iv.next = add nuw nsw i64 %iv, 1588 %exitcond.not = icmp eq i64 %iv.next, %n589 br i1 %exitcond.not, label %for.end, label %for.body590 591for.end:592 ret bfloat %add593}594 595; FMIN (FAST)596 597define float @fmin_fast(ptr noalias nocapture readonly %a, i64 %n) #0 {598; CHECK-LABEL: define float @fmin_fast(599; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR4:[0-9]+]] {600; CHECK-NEXT: [[ENTRY:.*:]]601; CHECK-NEXT: br label %[[VECTOR_PH:.*]]602; CHECK: [[VECTOR_PH]]:603; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]604; CHECK: [[VECTOR_BODY]]:605; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]606; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]607; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]608; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)609; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]610; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])611; CHECK-NEXT: [[TMP7:%.*]] = fcmp olt <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]612; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[VEC_PHI]]613; CHECK-NEXT: [[TMP9]] = call <vscale x 4 x float> @llvm.vp.merge.nxv4f32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> [[TMP8]], <vscale x 4 x float> [[VEC_PHI]], i32 [[TMP14]])614; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP14]] to i64615; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]616; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]617; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0618; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]619; CHECK: [[MIDDLE_BLOCK]]:620; CHECK-NEXT: [[TMP12:%.*]] = call float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> [[TMP9]])621; CHECK-NEXT: br label %[[FOR_END:.*]]622; CHECK: [[FOR_END]]:623; CHECK-NEXT: ret float [[TMP12]]624;625entry:626 br label %for.body627 628for.body:629 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]630 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %.sroa.speculated, %for.body ]631 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv632 %0 = load float, ptr %arrayidx, align 4633 %cmp.i = fcmp olt float %0, %sum.07634 %.sroa.speculated = select i1 %cmp.i, float %0, float %sum.07635 %iv.next = add nuw nsw i64 %iv, 1636 %exitcond.not = icmp eq i64 %iv.next, %n637 br i1 %exitcond.not, label %for.end, label %for.body638 639for.end:640 ret float %.sroa.speculated641}642 643define half @fmin_fast_half_zvfhmin(ptr noalias nocapture readonly %a, i64 %n) #1 {644; CHECK-LABEL: define half @fmin_fast_half_zvfhmin(645; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR5:[0-9]+]] {646; CHECK-NEXT: [[ENTRY:.*:]]647; CHECK-NEXT: br label %[[VECTOR_PH:.*]]648; CHECK: [[VECTOR_PH]]:649; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]650; CHECK: [[VECTOR_BODY]]:651; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]652; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 8 x half> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]653; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]654; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)655; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds half, ptr [[A]], i64 [[INDEX]]656; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 4 [[TMP6]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP14]])657; CHECK-NEXT: [[TMP7:%.*]] = fcmp olt <vscale x 8 x half> [[WIDE_LOAD]], [[VEC_PHI]]658; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 8 x i1> [[TMP7]], <vscale x 8 x half> [[WIDE_LOAD]], <vscale x 8 x half> [[VEC_PHI]]659; CHECK-NEXT: [[TMP9]] = call <vscale x 8 x half> @llvm.vp.merge.nxv8f16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x half> [[TMP8]], <vscale x 8 x half> [[VEC_PHI]], i32 [[TMP14]])660; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP14]] to i64661; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]662; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]663; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0664; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]665; CHECK: [[MIDDLE_BLOCK]]:666; CHECK-NEXT: [[TMP12:%.*]] = call half @llvm.vector.reduce.fmin.nxv8f16(<vscale x 8 x half> [[TMP9]])667; CHECK-NEXT: br label %[[FOR_END:.*]]668; CHECK: [[FOR_END]]:669; CHECK-NEXT: ret half [[TMP12]]670;671entry:672 br label %for.body673 674for.body:675 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]676 %sum.07 = phi half [ 0.000000e+00, %entry ], [ %.sroa.speculated, %for.body ]677 %arrayidx = getelementptr inbounds half, ptr %a, i64 %iv678 %0 = load half, ptr %arrayidx, align 4679 %cmp.i = fcmp olt half %0, %sum.07680 %.sroa.speculated = select i1 %cmp.i, half %0, half %sum.07681 %iv.next = add nuw nsw i64 %iv, 1682 %exitcond.not = icmp eq i64 %iv.next, %n683 br i1 %exitcond.not, label %for.end, label %for.body684 685for.end:686 ret half %.sroa.speculated687}688 689define bfloat @fmin_fast_bfloat_zvfbfmin(ptr noalias nocapture readonly %a, i64 %n) #2 {690; CHECK-LABEL: define bfloat @fmin_fast_bfloat_zvfbfmin(691; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR6:[0-9]+]] {692; CHECK-NEXT: [[ENTRY:.*:]]693; CHECK-NEXT: br label %[[VECTOR_PH:.*]]694; CHECK: [[VECTOR_PH]]:695; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]696; CHECK: [[VECTOR_BODY]]:697; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]698; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 8 x bfloat> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]699; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]700; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)701; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds bfloat, ptr [[A]], i64 [[INDEX]]702; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 8 x bfloat> @llvm.vp.load.nxv8bf16.p0(ptr align 4 [[TMP6]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP14]])703; CHECK-NEXT: [[TMP7:%.*]] = fcmp olt <vscale x 8 x bfloat> [[WIDE_LOAD]], [[VEC_PHI]]704; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 8 x i1> [[TMP7]], <vscale x 8 x bfloat> [[WIDE_LOAD]], <vscale x 8 x bfloat> [[VEC_PHI]]705; CHECK-NEXT: [[TMP9]] = call <vscale x 8 x bfloat> @llvm.vp.merge.nxv8bf16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x bfloat> [[TMP8]], <vscale x 8 x bfloat> [[VEC_PHI]], i32 [[TMP14]])706; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP14]] to i64707; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]708; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]709; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0710; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]711; CHECK: [[MIDDLE_BLOCK]]:712; CHECK-NEXT: [[TMP12:%.*]] = call bfloat @llvm.vector.reduce.fmin.nxv8bf16(<vscale x 8 x bfloat> [[TMP9]])713; CHECK-NEXT: br label %[[FOR_END:.*]]714; CHECK: [[FOR_END]]:715; CHECK-NEXT: ret bfloat [[TMP12]]716;717entry:718 br label %for.body719 720for.body:721 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]722 %sum.07 = phi bfloat [ 0.000000e+00, %entry ], [ %.sroa.speculated, %for.body ]723 %arrayidx = getelementptr inbounds bfloat, ptr %a, i64 %iv724 %0 = load bfloat, ptr %arrayidx, align 4725 %cmp.i = fcmp olt bfloat %0, %sum.07726 %.sroa.speculated = select i1 %cmp.i, bfloat %0, bfloat %sum.07727 %iv.next = add nuw nsw i64 %iv, 1728 %exitcond.not = icmp eq i64 %iv.next, %n729 br i1 %exitcond.not, label %for.end, label %for.body730 731for.end:732 ret bfloat %.sroa.speculated733}734 735; FMAX (FAST)736 737define float @fmax_fast(ptr noalias nocapture readonly %a, i64 %n) #0 {738; CHECK-LABEL: define float @fmax_fast(739; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR4]] {740; CHECK-NEXT: [[ENTRY:.*:]]741; CHECK-NEXT: br label %[[VECTOR_PH:.*]]742; CHECK: [[VECTOR_PH]]:743; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]744; CHECK: [[VECTOR_BODY]]:745; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]746; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]747; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]748; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)749; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]750; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])751; CHECK-NEXT: [[TMP7:%.*]] = fcmp fast ogt <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]752; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[VEC_PHI]]753; CHECK-NEXT: [[TMP9]] = call <vscale x 4 x float> @llvm.vp.merge.nxv4f32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> [[TMP8]], <vscale x 4 x float> [[VEC_PHI]], i32 [[TMP14]])754; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP14]] to i64755; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]756; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]757; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0758; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]759; CHECK: [[MIDDLE_BLOCK]]:760; CHECK-NEXT: [[TMP12:%.*]] = call fast float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> [[TMP9]])761; CHECK-NEXT: br label %[[FOR_END:.*]]762; CHECK: [[FOR_END]]:763; CHECK-NEXT: ret float [[TMP12]]764;765entry:766 br label %for.body767 768for.body:769 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]770 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %.sroa.speculated, %for.body ]771 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv772 %0 = load float, ptr %arrayidx, align 4773 %cmp.i = fcmp fast ogt float %0, %sum.07774 %.sroa.speculated = select i1 %cmp.i, float %0, float %sum.07775 %iv.next = add nuw nsw i64 %iv, 1776 %exitcond.not = icmp eq i64 %iv.next, %n777 br i1 %exitcond.not, label %for.end, label %for.body778 779for.end:780 ret float %.sroa.speculated781}782 783define half @fmax_fast_half_zvfhmin(ptr noalias nocapture readonly %a, i64 %n) #1 {784; CHECK-LABEL: define half @fmax_fast_half_zvfhmin(785; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR5]] {786; CHECK-NEXT: [[ENTRY:.*:]]787; CHECK-NEXT: br label %[[VECTOR_PH:.*]]788; CHECK: [[VECTOR_PH]]:789; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]790; CHECK: [[VECTOR_BODY]]:791; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]792; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 8 x half> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]793; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]794; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)795; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds half, ptr [[A]], i64 [[INDEX]]796; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 4 [[TMP6]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP14]])797; CHECK-NEXT: [[TMP7:%.*]] = fcmp fast ogt <vscale x 8 x half> [[WIDE_LOAD]], [[VEC_PHI]]798; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 8 x i1> [[TMP7]], <vscale x 8 x half> [[WIDE_LOAD]], <vscale x 8 x half> [[VEC_PHI]]799; CHECK-NEXT: [[TMP9]] = call <vscale x 8 x half> @llvm.vp.merge.nxv8f16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x half> [[TMP8]], <vscale x 8 x half> [[VEC_PHI]], i32 [[TMP14]])800; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP14]] to i64801; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]802; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]803; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0804; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]805; CHECK: [[MIDDLE_BLOCK]]:806; CHECK-NEXT: [[TMP12:%.*]] = call fast half @llvm.vector.reduce.fmax.nxv8f16(<vscale x 8 x half> [[TMP9]])807; CHECK-NEXT: br label %[[FOR_END:.*]]808; CHECK: [[FOR_END]]:809; CHECK-NEXT: ret half [[TMP12]]810;811entry:812 br label %for.body813 814for.body:815 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]816 %sum.07 = phi half [ 0.000000e+00, %entry ], [ %.sroa.speculated, %for.body ]817 %arrayidx = getelementptr inbounds half, ptr %a, i64 %iv818 %0 = load half, ptr %arrayidx, align 4819 %cmp.i = fcmp fast ogt half %0, %sum.07820 %.sroa.speculated = select i1 %cmp.i, half %0, half %sum.07821 %iv.next = add nuw nsw i64 %iv, 1822 %exitcond.not = icmp eq i64 %iv.next, %n823 br i1 %exitcond.not, label %for.end, label %for.body824 825for.end:826 ret half %.sroa.speculated827}828 829define bfloat @fmax_fast_bfloat_zvfbfmin(ptr noalias nocapture readonly %a, i64 %n) #2 {830; CHECK-LABEL: define bfloat @fmax_fast_bfloat_zvfbfmin(831; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR6]] {832; CHECK-NEXT: [[ENTRY:.*:]]833; CHECK-NEXT: br label %[[VECTOR_PH:.*]]834; CHECK: [[VECTOR_PH]]:835; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]836; CHECK: [[VECTOR_BODY]]:837; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]838; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 8 x bfloat> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]839; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]840; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)841; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds bfloat, ptr [[A]], i64 [[INDEX]]842; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 8 x bfloat> @llvm.vp.load.nxv8bf16.p0(ptr align 4 [[TMP6]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP14]])843; CHECK-NEXT: [[TMP7:%.*]] = fcmp fast ogt <vscale x 8 x bfloat> [[WIDE_LOAD]], [[VEC_PHI]]844; CHECK-NEXT: [[TMP8:%.*]] = select <vscale x 8 x i1> [[TMP7]], <vscale x 8 x bfloat> [[WIDE_LOAD]], <vscale x 8 x bfloat> [[VEC_PHI]]845; CHECK-NEXT: [[TMP9]] = call <vscale x 8 x bfloat> @llvm.vp.merge.nxv8bf16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x bfloat> [[TMP8]], <vscale x 8 x bfloat> [[VEC_PHI]], i32 [[TMP14]])846; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP14]] to i64847; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]848; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]849; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0850; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]851; CHECK: [[MIDDLE_BLOCK]]:852; CHECK-NEXT: [[TMP12:%.*]] = call fast bfloat @llvm.vector.reduce.fmax.nxv8bf16(<vscale x 8 x bfloat> [[TMP9]])853; CHECK-NEXT: br label %[[FOR_END:.*]]854; CHECK: [[FOR_END]]:855; CHECK-NEXT: ret bfloat [[TMP12]]856;857entry:858 br label %for.body859 860for.body:861 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]862 %sum.07 = phi bfloat [ 0.000000e+00, %entry ], [ %.sroa.speculated, %for.body ]863 %arrayidx = getelementptr inbounds bfloat, ptr %a, i64 %iv864 %0 = load bfloat, ptr %arrayidx, align 4865 %cmp.i = fcmp fast ogt bfloat %0, %sum.07866 %.sroa.speculated = select i1 %cmp.i, bfloat %0, bfloat %sum.07867 %iv.next = add nuw nsw i64 %iv, 1868 %exitcond.not = icmp eq i64 %iv.next, %n869 br i1 %exitcond.not, label %for.end, label %for.body870 871for.end:872 ret bfloat %.sroa.speculated873}874 875; Reduction cannot be vectorized876 877; MUL878 879define i32 @mul(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {880; CHECK-LABEL: define i32 @mul(881; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {882; CHECK-NEXT: [[ENTRY:.*]]:883; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16884; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]885; CHECK: [[VECTOR_PH]]:886; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16887; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]888; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]889; CHECK: [[VECTOR_BODY]]:890; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]891; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ <i32 2, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]892; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ splat (i32 1), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]893; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]894; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 8895; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP0]], align 4896; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4897; CHECK-NEXT: [[TMP2]] = mul <8 x i32> [[WIDE_LOAD]], [[VEC_PHI]]898; CHECK-NEXT: [[TMP3]] = mul <8 x i32> [[WIDE_LOAD2]], [[VEC_PHI1]]899; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16900; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]901; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]902; CHECK: [[MIDDLE_BLOCK]]:903; CHECK-NEXT: [[BIN_RDX:%.*]] = mul <8 x i32> [[TMP3]], [[TMP2]]904; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> [[BIN_RDX]])905; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]906; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]907; CHECK: [[SCALAR_PH]]:908; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]909; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ 2, %[[ENTRY]] ]910; CHECK-NEXT: br label %[[FOR_BODY:.*]]911; CHECK: [[FOR_BODY]]:912; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]913; CHECK-NEXT: [[SUM_07:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MUL:%.*]], %[[FOR_BODY]] ]914; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]915; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX]], align 4916; CHECK-NEXT: [[MUL]] = mul nsw i32 [[TMP6]], [[SUM_07]]917; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1918; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]919; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]920; CHECK: [[FOR_END]]:921; CHECK-NEXT: [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL]], %[[FOR_BODY]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ]922; CHECK-NEXT: ret i32 [[MUL_LCSSA]]923;924entry:925 br label %for.body926 927for.body: ; preds = %entry, %for.body928 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]929 %sum.07 = phi i32 [ 2, %entry ], [ %mul, %for.body ]930 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv931 %0 = load i32, ptr %arrayidx, align 4932 %mul = mul nsw i32 %0, %sum.07933 %iv.next = add nuw nsw i64 %iv, 1934 %exitcond.not = icmp eq i64 %iv.next, %n935 br i1 %exitcond.not, label %for.end, label %for.body936 937for.end: ; preds = %for.body, %entry938 ret i32 %mul939}940 941; Note: This test was added to ensure we always check the legality of reductions before checking for memory dependencies942define i32 @memory_dependence(ptr noalias nocapture %a, ptr noalias nocapture readonly %b, i64 %n) {943; CHECK-LABEL: define i32 @memory_dependence(944; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {945; CHECK-NEXT: [[ENTRY:.*]]:946; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8947; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]948; CHECK: [[VECTOR_PH]]:949; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8950; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]951; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]952; CHECK: [[VECTOR_BODY]]:953; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]954; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ <i32 2, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]955; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]956; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP0]], align 4957; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]958; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4959; CHECK-NEXT: [[TMP2:%.*]] = add nsw <8 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]960; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[INDEX]], 32961; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP3]]962; CHECK-NEXT: store <8 x i32> [[TMP2]], ptr [[TMP4]], align 4963; CHECK-NEXT: [[TMP5]] = mul <8 x i32> [[WIDE_LOAD1]], [[VEC_PHI]]964; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8965; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]966; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]967; CHECK: [[MIDDLE_BLOCK]]:968; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> [[TMP5]])969; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]970; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]971; CHECK: [[SCALAR_PH]]:972; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]973; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 2, %[[ENTRY]] ]974; CHECK-NEXT: br label %[[FOR_BODY:.*]]975; CHECK: [[FOR_BODY]]:976; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]977; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ [[MUL:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]978; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]979; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX]], align 4980; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I]]981; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4982; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP9]], [[TMP8]]983; CHECK-NEXT: [[ADD2:%.*]] = add nuw nsw i64 [[I]], 32984; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[ADD2]]985; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX3]], align 4986; CHECK-NEXT: [[MUL]] = mul nsw i32 [[TMP9]], [[SUM]]987; CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I]], 1988; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]989; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]990; CHECK: [[FOR_END]]:991; CHECK-NEXT: [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]992; CHECK-NEXT: ret i32 [[MUL_LCSSA]]993;994entry:995 br label %for.body996 997for.body:998 %i = phi i64 [ %inc, %for.body ], [ 0, %entry ]999 %sum = phi i32 [ %mul, %for.body ], [ 2, %entry ]1000 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %i1001 %0 = load i32, ptr %arrayidx, align 41002 %arrayidx1 = getelementptr inbounds i32, ptr %b, i64 %i1003 %1 = load i32, ptr %arrayidx1, align 41004 %add = add nsw i32 %1, %01005 %add2 = add nuw nsw i64 %i, 321006 %arrayidx3 = getelementptr inbounds i32, ptr %a, i64 %add21007 store i32 %add, ptr %arrayidx3, align 41008 %mul = mul nsw i32 %1, %sum1009 %inc = add nuw nsw i64 %i, 11010 %exitcond.not = icmp eq i64 %inc, %n1011 br i1 %exitcond.not, label %for.end, label %for.body1012 1013for.end:1014 ret i32 %mul1015}1016 1017define float @fmuladd(ptr %a, ptr %b, i64 %n) {1018; CHECK-LABEL: define float @fmuladd(1019; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {1020; CHECK-NEXT: [[ENTRY:.*:]]1021; CHECK-NEXT: br label %[[VECTOR_PH:.*]]1022; CHECK: [[VECTOR_PH]]:1023; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]1024; CHECK: [[VECTOR_BODY]]:1025; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]1026; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ insertelement (<vscale x 4 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]1027; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]1028; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)1029; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]1030; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])1031; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]1032; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])1033; CHECK-NEXT: [[TMP8:%.*]] = call reassoc <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[WIDE_LOAD1]], <vscale x 4 x float> [[VEC_PHI]])1034; CHECK-NEXT: [[TMP9]] = call <vscale x 4 x float> @llvm.vp.merge.nxv4f32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> [[TMP8]], <vscale x 4 x float> [[VEC_PHI]], i32 [[TMP14]])1035; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP14]] to i641036; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]1037; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]1038; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 01039; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]1040; CHECK: [[MIDDLE_BLOCK]]:1041; CHECK-NEXT: [[TMP16:%.*]] = call reassoc float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP9]])1042; CHECK-NEXT: br label %[[FOR_END:.*]]1043; CHECK: [[FOR_END]]:1044; CHECK-NEXT: ret float [[TMP16]]1045;1046entry:1047 br label %for.body1048 1049for.body:1050 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]1051 %sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd, %for.body ]1052 %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv1053 %0 = load float, ptr %arrayidx, align 41054 %arrayidx2 = getelementptr inbounds float, ptr %b, i64 %iv1055 %1 = load float, ptr %arrayidx2, align 41056 %muladd = tail call reassoc float @llvm.fmuladd.f32(float %0, float %1, float %sum.07)1057 %iv.next = add nuw nsw i64 %iv, 11058 %exitcond.not = icmp eq i64 %iv.next, %n1059 br i1 %exitcond.not, label %for.end, label %for.body1060 1061for.end:1062 ret float %muladd1063}1064 1065define half @fmuladd_f16_zvfh(ptr %a, ptr %b, i64 %n) "target-features"="+zvfh" {1066; CHECK-LABEL: define half @fmuladd_f16_zvfh(1067; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR1]] {1068; CHECK-NEXT: [[ENTRY:.*:]]1069; CHECK-NEXT: br label %[[VECTOR_PH:.*]]1070; CHECK: [[VECTOR_PH]]:1071; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]1072; CHECK: [[VECTOR_BODY]]:1073; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]1074; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 8 x half> [ insertelement (<vscale x 8 x half> splat (half 0xH8000), half 0xH0000, i32 0), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]1075; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]1076; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)1077; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds half, ptr [[A]], i64 [[INDEX]]1078; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 4 [[TMP6]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP14]])1079; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds half, ptr [[B]], i64 [[INDEX]]1080; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 4 [[TMP7]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP14]])1081; CHECK-NEXT: [[TMP8:%.*]] = call reassoc <vscale x 8 x half> @llvm.fmuladd.nxv8f16(<vscale x 8 x half> [[WIDE_LOAD]], <vscale x 8 x half> [[WIDE_LOAD1]], <vscale x 8 x half> [[VEC_PHI]])1082; CHECK-NEXT: [[TMP9]] = call <vscale x 8 x half> @llvm.vp.merge.nxv8f16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x half> [[TMP8]], <vscale x 8 x half> [[VEC_PHI]], i32 [[TMP14]])1083; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP14]] to i641084; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]1085; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]1086; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 01087; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]1088; CHECK: [[MIDDLE_BLOCK]]:1089; CHECK-NEXT: [[TMP16:%.*]] = call reassoc half @llvm.vector.reduce.fadd.nxv8f16(half 0xH8000, <vscale x 8 x half> [[TMP9]])1090; CHECK-NEXT: br label %[[FOR_END:.*]]1091; CHECK: [[FOR_END]]:1092; CHECK-NEXT: ret half [[TMP16]]1093;1094entry:1095 br label %for.body1096 1097for.body:1098 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]1099 %sum.07 = phi half [ 0.000000e+00, %entry ], [ %muladd, %for.body ]1100 %arrayidx = getelementptr inbounds half, ptr %a, i64 %iv1101 %0 = load half, ptr %arrayidx, align 41102 %arrayidx2 = getelementptr inbounds half, ptr %b, i64 %iv1103 %1 = load half, ptr %arrayidx2, align 41104 %muladd = tail call reassoc half @llvm.fmuladd.f16(half %0, half %1, half %sum.07)1105 %iv.next = add nuw nsw i64 %iv, 11106 %exitcond.not = icmp eq i64 %iv.next, %n1107 br i1 %exitcond.not, label %for.end, label %for.body1108 1109for.end:1110 ret half %muladd1111}1112 1113 1114; We can't scalably vectorize reductions of f16 with zvfhmin or bf16 with zvfbfmin, so make sure we use fixed-length vectors instead.1115 1116define half @fmuladd_f16_zvfhmin(ptr %a, ptr %b, i64 %n) "target-features"="+zvfhmin" {1117; CHECK-LABEL: define half @fmuladd_f16_zvfhmin(1118; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR2]] {1119; CHECK-NEXT: [[ENTRY:.*]]:1120; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 321121; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]1122; CHECK: [[VECTOR_PH]]:1123; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 321124; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]1125; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]1126; CHECK: [[VECTOR_BODY]]:1127; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]1128; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x half> [ <half 0xH0000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000, half 0xH8000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]1129; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <16 x half> [ splat (half 0xH8000), %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]1130; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds half, ptr [[A]], i64 [[INDEX]]1131; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds half, ptr [[TMP0]], i64 161132; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x half>, ptr [[TMP0]], align 41133; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x half>, ptr [[TMP1]], align 41134; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds half, ptr [[B]], i64 [[INDEX]]1135; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds half, ptr [[TMP2]], i64 161136; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x half>, ptr [[TMP2]], align 41137; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <16 x half>, ptr [[TMP3]], align 41138; CHECK-NEXT: [[TMP4]] = call reassoc <16 x half> @llvm.fmuladd.v16f16(<16 x half> [[WIDE_LOAD]], <16 x half> [[WIDE_LOAD3]], <16 x half> [[VEC_PHI]])1139; CHECK-NEXT: [[TMP5]] = call reassoc <16 x half> @llvm.fmuladd.v16f16(<16 x half> [[WIDE_LOAD2]], <16 x half> [[WIDE_LOAD4]], <16 x half> [[VEC_PHI1]])1140; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 321141; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1142; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]1143; CHECK: [[MIDDLE_BLOCK]]:1144; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd reassoc <16 x half> [[TMP5]], [[TMP4]]1145; CHECK-NEXT: [[TMP7:%.*]] = call reassoc half @llvm.vector.reduce.fadd.v16f16(half 0xH8000, <16 x half> [[BIN_RDX]])1146; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]1147; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]1148; CHECK: [[SCALAR_PH]]:1149; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]1150; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi half [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0xH0000, %[[ENTRY]] ]1151; CHECK-NEXT: br label %[[FOR_BODY:.*]]1152; CHECK: [[FOR_BODY]]:1153; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]1154; CHECK-NEXT: [[SUM_07:%.*]] = phi half [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MULADD:%.*]], %[[FOR_BODY]] ]1155; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds half, ptr [[A]], i64 [[IV]]1156; CHECK-NEXT: [[TMP8:%.*]] = load half, ptr [[ARRAYIDX]], align 41157; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds half, ptr [[B]], i64 [[IV]]1158; CHECK-NEXT: [[TMP9:%.*]] = load half, ptr [[ARRAYIDX2]], align 41159; CHECK-NEXT: [[MULADD]] = tail call reassoc half @llvm.fmuladd.f16(half [[TMP8]], half [[TMP9]], half [[SUM_07]])1160; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 11161; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]1162; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]1163; CHECK: [[FOR_END]]:1164; CHECK-NEXT: [[MULADD_LCSSA:%.*]] = phi half [ [[MULADD]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]1165; CHECK-NEXT: ret half [[MULADD_LCSSA]]1166;1167entry:1168 br label %for.body1169 1170for.body:1171 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]1172 %sum.07 = phi half [ 0.000000e+00, %entry ], [ %muladd, %for.body ]1173 %arrayidx = getelementptr inbounds half, ptr %a, i64 %iv1174 %0 = load half, ptr %arrayidx, align 41175 %arrayidx2 = getelementptr inbounds half, ptr %b, i64 %iv1176 %1 = load half, ptr %arrayidx2, align 41177 %muladd = tail call reassoc half @llvm.fmuladd.f16(half %0, half %1, half %sum.07)1178 %iv.next = add nuw nsw i64 %iv, 11179 %exitcond.not = icmp eq i64 %iv.next, %n1180 br i1 %exitcond.not, label %for.end, label %for.body1181 1182for.end:1183 ret half %muladd1184}1185 1186define bfloat @fmuladd_bf16(ptr %a, ptr %b, i64 %n) "target-features"="+zvfbfmin" {1187; CHECK-LABEL: define bfloat @fmuladd_bf16(1188; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR3]] {1189; CHECK-NEXT: [[ENTRY:.*]]:1190; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 321191; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]1192; CHECK: [[VECTOR_PH]]:1193; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 321194; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]1195; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]1196; CHECK: [[VECTOR_BODY]]:1197; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]1198; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x bfloat> [ <bfloat 0xR0000, bfloat 0xR8000, bfloat 0xR8000, bfloat 0xR8000, bfloat 0xR8000, bfloat 0xR8000, bfloat 0xR8000, bfloat 0xR8000, bfloat 0xR8000, bfloat 0xR8000, bfloat 0xR8000, bfloat 0xR8000, bfloat 0xR8000, bfloat 0xR8000, bfloat 0xR8000, bfloat 0xR8000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]1199; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <16 x bfloat> [ splat (bfloat 0xR8000), %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]1200; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds bfloat, ptr [[A]], i64 [[INDEX]]1201; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds bfloat, ptr [[TMP0]], i64 161202; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x bfloat>, ptr [[TMP0]], align 41203; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x bfloat>, ptr [[TMP1]], align 41204; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds bfloat, ptr [[B]], i64 [[INDEX]]1205; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds bfloat, ptr [[TMP2]], i64 161206; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <16 x bfloat>, ptr [[TMP2]], align 41207; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <16 x bfloat>, ptr [[TMP3]], align 41208; CHECK-NEXT: [[TMP4]] = call reassoc <16 x bfloat> @llvm.fmuladd.v16bf16(<16 x bfloat> [[WIDE_LOAD]], <16 x bfloat> [[WIDE_LOAD3]], <16 x bfloat> [[VEC_PHI]])1209; CHECK-NEXT: [[TMP5]] = call reassoc <16 x bfloat> @llvm.fmuladd.v16bf16(<16 x bfloat> [[WIDE_LOAD2]], <16 x bfloat> [[WIDE_LOAD4]], <16 x bfloat> [[VEC_PHI1]])1210; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 321211; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]1212; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]1213; CHECK: [[MIDDLE_BLOCK]]:1214; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd reassoc <16 x bfloat> [[TMP5]], [[TMP4]]1215; CHECK-NEXT: [[TMP7:%.*]] = call reassoc bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0xR8000, <16 x bfloat> [[BIN_RDX]])1216; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]1217; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]1218; CHECK: [[SCALAR_PH]]:1219; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]1220; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi bfloat [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0xR0000, %[[ENTRY]] ]1221; CHECK-NEXT: br label %[[FOR_BODY:.*]]1222; CHECK: [[FOR_BODY]]:1223; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]1224; CHECK-NEXT: [[SUM_07:%.*]] = phi bfloat [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MULADD:%.*]], %[[FOR_BODY]] ]1225; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds bfloat, ptr [[A]], i64 [[IV]]1226; CHECK-NEXT: [[TMP8:%.*]] = load bfloat, ptr [[ARRAYIDX]], align 41227; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds bfloat, ptr [[B]], i64 [[IV]]1228; CHECK-NEXT: [[TMP9:%.*]] = load bfloat, ptr [[ARRAYIDX2]], align 41229; CHECK-NEXT: [[MULADD]] = tail call reassoc bfloat @llvm.fmuladd.bf16(bfloat [[TMP8]], bfloat [[TMP9]], bfloat [[SUM_07]])1230; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 11231; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]1232; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]1233; CHECK: [[FOR_END]]:1234; CHECK-NEXT: [[MULADD_LCSSA:%.*]] = phi bfloat [ [[MULADD]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]1235; CHECK-NEXT: ret bfloat [[MULADD_LCSSA]]1236;1237entry:1238 br label %for.body1239 1240for.body:1241 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]1242 %sum.07 = phi bfloat [ 0.000000e+00, %entry ], [ %muladd, %for.body ]1243 %arrayidx = getelementptr inbounds bfloat, ptr %a, i64 %iv1244 %0 = load bfloat, ptr %arrayidx, align 41245 %arrayidx2 = getelementptr inbounds bfloat, ptr %b, i64 %iv1246 %1 = load bfloat, ptr %arrayidx2, align 41247 %muladd = tail call reassoc bfloat @llvm.fmuladd.bf16(bfloat %0, bfloat %1, bfloat %sum.07)1248 %iv.next = add nuw nsw i64 %iv, 11249 %exitcond.not = icmp eq i64 %iv.next, %n1250 br i1 %exitcond.not, label %for.end, label %for.body1251 1252for.end:1253 ret bfloat %muladd1254}1255 1256declare float @llvm.fmuladd.f32(float, float, float)1257 1258attributes #0 = { "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" }1259attributes #1 = { "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "target-features"="+zfhmin,+zvfhmin"}1260attributes #2 = { "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "target-features"="+zfbfmin,+zvfbfmin"}1261