389 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=armv8.1m.main -mattr=+mve -tail-predication=enabled --verify-machineinstrs %s -o - | FileCheck %s3 4define dso_local i32 @mul_reduce_add(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, i32 %N) {5; CHECK-LABEL: mul_reduce_add:6; CHECK: @ %bb.0: @ %entry7; CHECK-NEXT: cmp r2, #08; CHECK-NEXT: itt eq9; CHECK-NEXT: moveq r0, #010; CHECK-NEXT: bxeq lr11; CHECK-NEXT: .LBB0_1: @ %vector.ph12; CHECK-NEXT: push {r7, lr}13; CHECK-NEXT: adds r3, r2, #314; CHECK-NEXT: vmov.i32 q1, #0x015; CHECK-NEXT: bic r3, r3, #316; CHECK-NEXT: sub.w r12, r3, #417; CHECK-NEXT: movs r3, #118; CHECK-NEXT: add.w r3, r3, r12, lsr #219; CHECK-NEXT: dls lr, r320; CHECK-NEXT: .LBB0_2: @ %vector.body21; CHECK-NEXT: @ =>This Inner Loop Header: Depth=122; CHECK-NEXT: vctp.32 r223; CHECK-NEXT: vmov q0, q124; CHECK-NEXT: vpstt25; CHECK-NEXT: vldrwt.u32 q1, [r0], #1626; CHECK-NEXT: vldrwt.u32 q2, [r1], #1627; CHECK-NEXT: subs r2, #428; CHECK-NEXT: vmul.i32 q1, q2, q129; CHECK-NEXT: vadd.i32 q1, q1, q030; CHECK-NEXT: le lr, .LBB0_231; CHECK-NEXT: @ %bb.3: @ %middle.block32; CHECK-NEXT: vpsel q0, q1, q033; CHECK-NEXT: vaddv.u32 r0, q034; CHECK-NEXT: pop {r7, pc}35entry:36 %cmp8 = icmp eq i32 %N, 037 br i1 %cmp8, label %for.cond.cleanup, label %vector.ph38 39vector.ph: ; preds = %entry40 %n.rnd.up = add i32 %N, 341 %n.vec = and i32 %n.rnd.up, -442 br label %vector.body43 44vector.body: ; preds = %vector.body, %vector.ph45 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]46 %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %6, %vector.body ]47 %0 = getelementptr inbounds i32, ptr %a, i32 %index48 %1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)49 %2 = bitcast ptr %0 to ptr50 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %2, i32 4, <4 x i1> %1, <4 x i32> undef)51 %3 = getelementptr inbounds i32, ptr %b, i32 %index52 %4 = bitcast ptr %3 to ptr53 %wide.masked.load13 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %4, i32 4, <4 x i1> %1, <4 x i32> undef)54 %5 = mul nsw <4 x i32> %wide.masked.load13, %wide.masked.load55 %6 = add nsw <4 x i32> %5, %vec.phi56 %index.next = add i32 %index, 457 %7 = icmp eq i32 %index.next, %n.vec58 br i1 %7, label %middle.block, label %vector.body59 60middle.block: ; preds = %vector.body61 %8 = select <4 x i1> %1, <4 x i32> %6, <4 x i32> %vec.phi62 %9 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %8)63 br label %for.cond.cleanup64 65for.cond.cleanup: ; preds = %middle.block, %entry66 %res.0.lcssa = phi i32 [ 0, %entry ], [ %9, %middle.block ]67 ret i32 %res.0.lcssa68}69 70define dso_local i32 @mul_reduce_add_const(ptr noalias nocapture readonly %a, i32 %b, i32 %N) {71; CHECK-LABEL: mul_reduce_add_const:72; CHECK: @ %bb.0: @ %entry73; CHECK-NEXT: cmp r2, #074; CHECK-NEXT: itt eq75; CHECK-NEXT: moveq r0, #076; CHECK-NEXT: bxeq lr77; CHECK-NEXT: .LBB1_1: @ %vector.ph78; CHECK-NEXT: push {r7, lr}79; CHECK-NEXT: adds r1, r2, #380; CHECK-NEXT: movs r3, #181; CHECK-NEXT: bic r1, r1, #382; CHECK-NEXT: vmov.i32 q0, #0x083; CHECK-NEXT: subs r1, #484; CHECK-NEXT: add.w r1, r3, r1, lsr #285; CHECK-NEXT: dls lr, r186; CHECK-NEXT: .LBB1_2: @ %vector.body87; CHECK-NEXT: @ =>This Inner Loop Header: Depth=188; CHECK-NEXT: vctp.32 r289; CHECK-NEXT: vmov q1, q090; CHECK-NEXT: vpst91; CHECK-NEXT: vldrwt.u32 q0, [r0], #1692; CHECK-NEXT: subs r2, #493; CHECK-NEXT: vadd.i32 q0, q0, q194; CHECK-NEXT: le lr, .LBB1_295; CHECK-NEXT: @ %bb.3: @ %middle.block96; CHECK-NEXT: vpsel q0, q0, q197; CHECK-NEXT: vaddv.u32 r0, q098; CHECK-NEXT: pop {r7, pc}99entry:100 %cmp6 = icmp eq i32 %N, 0101 br i1 %cmp6, label %for.cond.cleanup, label %vector.ph102 103vector.ph: ; preds = %entry104 %n.rnd.up = add i32 %N, 3105 %n.vec = and i32 %n.rnd.up, -4106 br label %vector.body107 108vector.body: ; preds = %vector.body, %vector.ph109 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]110 %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]111 %0 = getelementptr inbounds i32, ptr %a, i32 %index112 %1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)113 %2 = bitcast ptr %0 to ptr114 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %2, i32 4, <4 x i1> %1, <4 x i32> undef)115 %3 = add nsw <4 x i32> %wide.masked.load, %vec.phi116 %index.next = add i32 %index, 4117 %4 = icmp eq i32 %index.next, %n.vec118 br i1 %4, label %middle.block, label %vector.body119 120middle.block: ; preds = %vector.body121 %5 = select <4 x i1> %1, <4 x i32> %3, <4 x i32> %vec.phi122 %6 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %5)123 br label %for.cond.cleanup124 125for.cond.cleanup: ; preds = %middle.block, %entry126 %res.0.lcssa = phi i32 [ 0, %entry ], [ %6, %middle.block ]127 ret i32 %res.0.lcssa128}129 130define dso_local i32 @add_reduce_add_const(ptr noalias nocapture readonly %a, i32 %b, i32 %N) {131; CHECK-LABEL: add_reduce_add_const:132; CHECK: @ %bb.0: @ %entry133; CHECK-NEXT: cmp r2, #0134; CHECK-NEXT: itt eq135; CHECK-NEXT: moveq r0, #0136; CHECK-NEXT: bxeq lr137; CHECK-NEXT: .LBB2_1: @ %vector.ph138; CHECK-NEXT: push {r7, lr}139; CHECK-NEXT: adds r1, r2, #3140; CHECK-NEXT: movs r3, #1141; CHECK-NEXT: bic r1, r1, #3142; CHECK-NEXT: vmov.i32 q0, #0x0143; CHECK-NEXT: subs r1, #4144; CHECK-NEXT: add.w r1, r3, r1, lsr #2145; CHECK-NEXT: dls lr, r1146; CHECK-NEXT: .LBB2_2: @ %vector.body147; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1148; CHECK-NEXT: vctp.32 r2149; CHECK-NEXT: vmov q1, q0150; CHECK-NEXT: vpst151; CHECK-NEXT: vldrwt.u32 q0, [r0], #16152; CHECK-NEXT: subs r2, #4153; CHECK-NEXT: vadd.i32 q0, q0, q1154; CHECK-NEXT: le lr, .LBB2_2155; CHECK-NEXT: @ %bb.3: @ %middle.block156; CHECK-NEXT: vpsel q0, q0, q1157; CHECK-NEXT: vaddv.u32 r0, q0158; CHECK-NEXT: pop {r7, pc}159entry:160 %cmp6 = icmp eq i32 %N, 0161 br i1 %cmp6, label %for.cond.cleanup, label %vector.ph162 163vector.ph: ; preds = %entry164 %n.rnd.up = add i32 %N, 3165 %n.vec = and i32 %n.rnd.up, -4166 br label %vector.body167 168vector.body: ; preds = %vector.body, %vector.ph169 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]170 %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]171 %0 = getelementptr inbounds i32, ptr %a, i32 %index172 %1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)173 %2 = bitcast ptr %0 to ptr174 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %2, i32 4, <4 x i1> %1, <4 x i32> undef)175 %3 = add nsw <4 x i32> %wide.masked.load, %vec.phi176 %index.next = add i32 %index, 4177 %4 = icmp eq i32 %index.next, %n.vec178 br i1 %4, label %middle.block, label %vector.body179 180middle.block: ; preds = %vector.body181 %5 = select <4 x i1> %1, <4 x i32> %3, <4 x i32> %vec.phi182 %6 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %5)183 br label %for.cond.cleanup184 185for.cond.cleanup: ; preds = %middle.block, %entry186 %res.0.lcssa = phi i32 [ 0, %entry ], [ %6, %middle.block ]187 ret i32 %res.0.lcssa188}189 190define dso_local void @vector_mul_const(ptr noalias nocapture %a, ptr noalias nocapture readonly %b, i32 %c, i32 %N) {191; CHECK-LABEL: vector_mul_const:192; CHECK: @ %bb.0: @ %entry193; CHECK-NEXT: push {r7, lr}194; CHECK-NEXT: cmp r3, #0195; CHECK-NEXT: it eq196; CHECK-NEXT: popeq {r7, pc}197; CHECK-NEXT: .LBB3_1: @ %vector.ph198; CHECK-NEXT: dlstp.32 lr, r3199; CHECK-NEXT: .LBB3_2: @ %vector.body200; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1201; CHECK-NEXT: vldrw.u32 q0, [r1], #16202; CHECK-NEXT: vmul.i32 q0, q0, r2203; CHECK-NEXT: vstrw.32 q0, [r0], #16204; CHECK-NEXT: letp lr, .LBB3_2205; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup206; CHECK-NEXT: pop {r7, pc}207entry:208 %cmp6 = icmp eq i32 %N, 0209 br i1 %cmp6, label %for.cond.cleanup, label %vector.ph210 211vector.ph: ; preds = %entry212 %n.rnd.up = add i32 %N, 3213 %n.vec = and i32 %n.rnd.up, -4214 %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %c, i32 0215 %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer216 br label %vector.body217 218vector.body: ; preds = %vector.body, %vector.ph219 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]220 %0 = getelementptr inbounds i32, ptr %b, i32 %index221 %1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)222 %2 = bitcast ptr %0 to ptr223 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %2, i32 4, <4 x i1> %1, <4 x i32> undef)224 %3 = mul nsw <4 x i32> %wide.masked.load, %broadcast.splat11225 %4 = getelementptr inbounds i32, ptr %a, i32 %index226 %5 = bitcast ptr %4 to ptr227 call void @llvm.masked.store.v4i32.p0(<4 x i32> %3, ptr %5, i32 4, <4 x i1> %1)228 %index.next = add i32 %index, 4229 %6 = icmp eq i32 %index.next, %n.vec230 br i1 %6, label %for.cond.cleanup, label %vector.body231 232for.cond.cleanup: ; preds = %vector.body, %entry233 ret void234}235 236define dso_local void @vector_add_const(ptr noalias nocapture %a, ptr noalias nocapture readonly %b, i32 %c, i32 %N) {237; CHECK-LABEL: vector_add_const:238; CHECK: @ %bb.0: @ %entry239; CHECK-NEXT: push {r7, lr}240; CHECK-NEXT: cmp r3, #0241; CHECK-NEXT: it eq242; CHECK-NEXT: popeq {r7, pc}243; CHECK-NEXT: .LBB4_1: @ %vector.ph244; CHECK-NEXT: dlstp.32 lr, r3245; CHECK-NEXT: .LBB4_2: @ %vector.body246; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1247; CHECK-NEXT: vldrw.u32 q0, [r1], #16248; CHECK-NEXT: vadd.i32 q0, q0, r2249; CHECK-NEXT: vstrw.32 q0, [r0], #16250; CHECK-NEXT: letp lr, .LBB4_2251; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup252; CHECK-NEXT: pop {r7, pc}253entry:254 %cmp6 = icmp eq i32 %N, 0255 br i1 %cmp6, label %for.cond.cleanup, label %vector.ph256 257vector.ph: ; preds = %entry258 %n.rnd.up = add i32 %N, 3259 %n.vec = and i32 %n.rnd.up, -4260 %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %c, i32 0261 %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer262 br label %vector.body263 264vector.body: ; preds = %vector.body, %vector.ph265 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]266 %0 = getelementptr inbounds i32, ptr %b, i32 %index267 %1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)268 %2 = bitcast ptr %0 to ptr269 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %2, i32 4, <4 x i1> %1, <4 x i32> undef)270 %3 = add nsw <4 x i32> %wide.masked.load, %broadcast.splat11271 %4 = getelementptr inbounds i32, ptr %a, i32 %index272 %5 = bitcast ptr %4 to ptr273 call void @llvm.masked.store.v4i32.p0(<4 x i32> %3, ptr %5, i32 4, <4 x i1> %1)274 %index.next = add i32 %index, 4275 %6 = icmp eq i32 %index.next, %n.vec276 br i1 %6, label %for.cond.cleanup, label %vector.body277 278for.cond.cleanup: ; preds = %vector.body, %entry279 ret void280}281 282define dso_local arm_aapcs_vfpcc void @vector_mul_vector_i8(ptr noalias nocapture %a, ptr noalias nocapture readonly %b, ptr noalias nocapture readonly %c, i32 %N) {283; CHECK-LABEL: vector_mul_vector_i8:284; CHECK: @ %bb.0: @ %entry285; CHECK-NEXT: push {r7, lr}286; CHECK-NEXT: cmp r3, #0287; CHECK-NEXT: it eq288; CHECK-NEXT: popeq {r7, pc}289; CHECK-NEXT: .LBB5_1: @ %vector.ph290; CHECK-NEXT: dlstp.8 lr, r3291; CHECK-NEXT: .LBB5_2: @ %vector.body292; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1293; CHECK-NEXT: vldrb.u8 q0, [r1], #16294; CHECK-NEXT: vldrb.u8 q1, [r2], #16295; CHECK-NEXT: vmul.i8 q0, q1, q0296; CHECK-NEXT: vstrb.8 q0, [r0], #16297; CHECK-NEXT: letp lr, .LBB5_2298; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup299; CHECK-NEXT: pop {r7, pc}300entry:301 %cmp10 = icmp eq i32 %N, 0302 br i1 %cmp10, label %for.cond.cleanup, label %vector.ph303 304vector.ph: ; preds = %entry305 %n.rnd.up = add i32 %N, 15306 %n.vec = and i32 %n.rnd.up, -16307 br label %vector.body308 309vector.body: ; preds = %vector.body, %vector.ph310 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]311 %0 = getelementptr inbounds i8, ptr %b, i32 %index312 %1 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %N)313 %2 = bitcast ptr %0 to ptr314 %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %2, i32 1, <16 x i1> %1, <16 x i8> undef)315 %3 = getelementptr inbounds i8, ptr %c, i32 %index316 %4 = bitcast ptr %3 to ptr317 %wide.masked.load14 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %4, i32 1, <16 x i1> %1, <16 x i8> undef)318 %5 = mul <16 x i8> %wide.masked.load14, %wide.masked.load319 %6 = getelementptr inbounds i8, ptr %a, i32 %index320 %7 = bitcast ptr %6 to ptr321 call void @llvm.masked.store.v16i8.p0(<16 x i8> %5, ptr %7, i32 1, <16 x i1> %1)322 %index.next = add i32 %index, 16323 %8 = icmp eq i32 %index.next, %n.vec324 br i1 %8, label %for.cond.cleanup, label %vector.body325 326for.cond.cleanup: ; preds = %vector.body, %entry327 ret void328}329 330; Function Attrs: nofree norecurse nounwind331define dso_local arm_aapcs_vfpcc void @vector_mul_vector_i16(ptr noalias nocapture %a, ptr noalias nocapture readonly %b, ptr noalias nocapture readonly %c, i32 %N) local_unnamed_addr #0 {332; CHECK-LABEL: vector_mul_vector_i16:333; CHECK: @ %bb.0: @ %entry334; CHECK-NEXT: push {r7, lr}335; CHECK-NEXT: cmp r3, #0336; CHECK-NEXT: it eq337; CHECK-NEXT: popeq {r7, pc}338; CHECK-NEXT: .LBB6_1: @ %vector.ph339; CHECK-NEXT: dlstp.16 lr, r3340; CHECK-NEXT: .LBB6_2: @ %vector.body341; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1342; CHECK-NEXT: vldrh.u16 q0, [r1], #16343; CHECK-NEXT: vldrh.u16 q1, [r2], #16344; CHECK-NEXT: vmul.i16 q0, q1, q0345; CHECK-NEXT: vstrh.16 q0, [r0], #16346; CHECK-NEXT: letp lr, .LBB6_2347; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup348; CHECK-NEXT: pop {r7, pc}349entry:350 %cmp10 = icmp eq i32 %N, 0351 br i1 %cmp10, label %for.cond.cleanup, label %vector.ph352 353vector.ph: ; preds = %entry354 %n.rnd.up = add i32 %N, 7355 %n.vec = and i32 %n.rnd.up, -8356 br label %vector.body357 358vector.body: ; preds = %vector.body, %vector.ph359 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]360 %0 = getelementptr inbounds i16, ptr %b, i32 %index361 %1 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %N)362 %2 = bitcast ptr %0 to ptr363 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %2, i32 2, <8 x i1> %1, <8 x i16> undef)364 %3 = getelementptr inbounds i16, ptr %c, i32 %index365 %4 = bitcast ptr %3 to ptr366 %wide.masked.load14 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %4, i32 2, <8 x i1> %1, <8 x i16> undef)367 %5 = mul <8 x i16> %wide.masked.load14, %wide.masked.load368 %6 = getelementptr inbounds i16, ptr %a, i32 %index369 %7 = bitcast ptr %6 to ptr370 call void @llvm.masked.store.v8i16.p0(<8 x i16> %5, ptr %7, i32 2, <8 x i1> %1)371 %index.next = add i32 %index, 8372 %8 = icmp eq i32 %index.next, %n.vec373 br i1 %8, label %for.cond.cleanup, label %vector.body374 375for.cond.cleanup: ; preds = %vector.body, %entry376 ret void377}378 379declare <16 x i8> @llvm.masked.load.v16i8.p0(ptr, i32 immarg, <16 x i1>, <16 x i8>)380declare <8 x i16> @llvm.masked.load.v8i16.p0(ptr, i32 immarg, <8 x i1>, <8 x i16>)381declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32 immarg, <4 x i1>, <4 x i32>)382declare void @llvm.masked.store.v16i8.p0(<16 x i8>, ptr, i32 immarg, <16 x i1>)383declare void @llvm.masked.store.v8i16.p0(<8 x i16>, ptr, i32 immarg, <8 x i1>)384declare void @llvm.masked.store.v4i32.p0(<4 x i32>, ptr, i32 immarg, <4 x i1>)385declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)386declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)387declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32, i32)388declare <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32, i32)389