303 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s3 4define arm_aapcs_vfpcc i16 @reduce_v16i16_shift_mul(<16 x i8> %s0, <16 x i8> %s1) {5; CHECK-LABEL: reduce_v16i16_shift_mul:6; CHECK: @ %bb.0: @ %entry7; CHECK-NEXT: vmullt.u8 q2, q0, q18; CHECK-NEXT: vmullb.u8 q0, q0, q19; CHECK-NEXT: vshr.s16 q2, q2, #1410; CHECK-NEXT: vshr.s16 q0, q0, #1411; CHECK-NEXT: vaddv.u16 r0, q212; CHECK-NEXT: vaddva.u16 r0, q013; CHECK-NEXT: bx lr14entry:15 %s0s = zext <16 x i8> %s0 to <16 x i16>16 %s1s = zext <16 x i8> %s1 to <16 x i16>17 %m = mul <16 x i16> %s0s, %s1s18 %sh = ashr <16 x i16> %m, <i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14>19 %result = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %sh)20 ret i16 %result21}22 23define arm_aapcs_vfpcc i16 @reduce_v8i16_shift_mul(<8 x i8> %s0, <8 x i8> %s1) {24; CHECK-LABEL: reduce_v8i16_shift_mul:25; CHECK: @ %bb.0: @ %entry26; CHECK-NEXT: vmullb.u8 q0, q0, q127; CHECK-NEXT: vshr.s16 q0, q0, #1428; CHECK-NEXT: vaddv.u16 r0, q029; CHECK-NEXT: bx lr30entry:31 %s0s = zext <8 x i8> %s0 to <8 x i16>32 %s1s = zext <8 x i8> %s1 to <8 x i16>33 %m = mul <8 x i16> %s0s, %s1s34 %sh = ashr <8 x i16> %m, <i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14>35 %result = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %sh)36 ret i16 %result37}38 39define arm_aapcs_vfpcc i16 @reduce_v16i16_shift_sub(<16 x i8> %s0, <16 x i8> %s1) {40; CHECK-LABEL: reduce_v16i16_shift_sub:41; CHECK: @ %bb.0: @ %entry42; CHECK-NEXT: vmovlt.u8 q2, q143; CHECK-NEXT: vmovlt.u8 q3, q044; CHECK-NEXT: vsub.i16 q2, q3, q245; CHECK-NEXT: vmovlb.u8 q1, q146; CHECK-NEXT: vmovlb.u8 q0, q047; CHECK-NEXT: vshr.s16 q2, q2, #1448; CHECK-NEXT: vsub.i16 q0, q0, q149; CHECK-NEXT: vaddv.u16 r0, q250; CHECK-NEXT: vshr.s16 q0, q0, #1451; CHECK-NEXT: vaddva.u16 r0, q052; CHECK-NEXT: bx lr53entry:54 %s0s = zext <16 x i8> %s0 to <16 x i16>55 %s1s = zext <16 x i8> %s1 to <16 x i16>56 %m = sub <16 x i16> %s0s, %s1s57 %sh = ashr <16 x i16> %m, <i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14, i16 14>58 %result = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %sh)59 ret i16 %result60}61 62define arm_aapcs_vfpcc i32 @mlapred_v4i32_v4i64_zext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %a, <8 x i16> %b) {63; CHECK-LABEL: mlapred_v4i32_v4i64_zext:64; CHECK: @ %bb.0: @ %entry65; CHECK-NEXT: .vsave {d8, d9, d10, d11}66; CHECK-NEXT: vpush {d8, d9, d10, d11}67; CHECK-NEXT: .pad #3268; CHECK-NEXT: sub sp, #3269; CHECK-NEXT: vorr q2, q2, q370; CHECK-NEXT: mov r0, sp71; CHECK-NEXT: vstrw.32 q2, [r0]72; CHECK-NEXT: vmov.i8 q3, #0xff73; CHECK-NEXT: vldrh.u32 q2, [r0, #8]74; CHECK-NEXT: vldrh.u32 q5, [r0]75; CHECK-NEXT: add r0, sp, #1676; CHECK-NEXT: vcmp.i32 eq, q2, zr77; CHECK-NEXT: vmov.i8 q2, #0x078; CHECK-NEXT: vpsel q4, q3, q279; CHECK-NEXT: vcmp.i32 eq, q5, zr80; CHECK-NEXT: vpsel q2, q3, q281; CHECK-NEXT: vstrh.32 q4, [r0, #8]82; CHECK-NEXT: vstrh.32 q2, [r0]83; CHECK-NEXT: vldrw.u32 q2, [r0]84; CHECK-NEXT: vpt.i16 ne, q2, zr85; CHECK-NEXT: vmlavt.u16 r0, q0, q186; CHECK-NEXT: add sp, #3287; CHECK-NEXT: vpop {d8, d9, d10, d11}88; CHECK-NEXT: bx lr89entry:90 %aa = zext <8 x i16> %a to <8 x i32>91 %bb = zext <8 x i16> %b to <8 x i32>92 %c1 = icmp eq <8 x i32> %aa, zeroinitializer93 %c2 = icmp eq <8 x i32> %bb, zeroinitializer94 %c = and <8 x i1> %c1, %c295 %xx = zext <8 x i16> %x to <8 x i32>96 %yy = zext <8 x i16> %y to <8 x i32>97 %m = mul <8 x i32> %xx, %yy98 %s = select <8 x i1> %c, <8 x i32> %m, <8 x i32> zeroinitializer99 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %s)100 ret i32 %z101}102 103define void @correlate(ptr nocapture noundef readonly %ID, ptr nocapture noundef writeonly %ACD, i16 noundef signext %DS, i16 noundef signext %Ls, i16 noundef signext %S) {104; CHECK-LABEL: correlate:105; CHECK: @ %bb.0: @ %entry106; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}107; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}108; CHECK-NEXT: .pad #8109; CHECK-NEXT: sub sp, #8110; CHECK-NEXT: cmp r3, #1111; CHECK-NEXT: str r0, [sp] @ 4-byte Spill112; CHECK-NEXT: blt .LBB4_12113; CHECK-NEXT: @ %bb.1: @ %for.body.lr.ph114; CHECK-NEXT: ldr r7, [sp, #44]115; CHECK-NEXT: add.w r10, r2, #3116; CHECK-NEXT: ldr.w r11, [sp] @ 4-byte Reload117; CHECK-NEXT: mov r9, r2118; CHECK-NEXT: mov r5, r1119; CHECK-NEXT: mov r0, r2120; CHECK-NEXT: uxth.w r12, r7121; CHECK-NEXT: mov.w r8, #0122; CHECK-NEXT: str.w r9, [sp, #4] @ 4-byte Spill123; CHECK-NEXT: b .LBB4_4124; CHECK-NEXT: .LBB4_2: @ in Loop: Header=BB4_4 Depth=1125; CHECK-NEXT: movs r6, #0126; CHECK-NEXT: .LBB4_3: @ %for.end127; CHECK-NEXT: @ in Loop: Header=BB4_4 Depth=1128; CHECK-NEXT: lsrs r0, r6, #16129; CHECK-NEXT: sub.w r10, r10, #1130; CHECK-NEXT: strh.w r0, [r5, r8, lsl #1]131; CHECK-NEXT: add.w r8, r8, #1132; CHECK-NEXT: add.w r11, r11, #2133; CHECK-NEXT: ldr r0, [sp, #4] @ 4-byte Reload134; CHECK-NEXT: sub.w r9, r9, #1135; CHECK-NEXT: cmp r8, r3136; CHECK-NEXT: beq .LBB4_12137; CHECK-NEXT: .LBB4_4: @ %for.body138; CHECK-NEXT: @ =>This Loop Header: Depth=1139; CHECK-NEXT: @ Child Loop BB4_8 Depth 2140; CHECK-NEXT: @ Child Loop BB4_11 Depth 2141; CHECK-NEXT: cmp r0, r8142; CHECK-NEXT: ble .LBB4_2143; CHECK-NEXT: @ %bb.5: @ %vector.main.loop.iter.check144; CHECK-NEXT: @ in Loop: Header=BB4_4 Depth=1145; CHECK-NEXT: sub.w r0, r0, r8146; CHECK-NEXT: mov r2, r5147; CHECK-NEXT: cmp r0, #8148; CHECK-NEXT: bhs .LBB4_7149; CHECK-NEXT: @ %bb.6: @ in Loop: Header=BB4_4 Depth=1150; CHECK-NEXT: movs r6, #0151; CHECK-NEXT: movs r1, #0152; CHECK-NEXT: mov r5, r2153; CHECK-NEXT: b .LBB4_10154; CHECK-NEXT: .LBB4_7: @ %vector.ph155; CHECK-NEXT: @ in Loop: Header=BB4_4 Depth=1156; CHECK-NEXT: bic r7, r9, #7157; CHECK-NEXT: movs r6, #1158; CHECK-NEXT: subs r7, #8159; CHECK-NEXT: bic r1, r0, #7160; CHECK-NEXT: mov r5, r11161; CHECK-NEXT: add.w lr, r6, r7, lsr #3162; CHECK-NEXT: movs r6, #0163; CHECK-NEXT: ldr r4, [sp] @ 4-byte Reload164; CHECK-NEXT: .LBB4_8: @ %vector.body165; CHECK-NEXT: @ Parent Loop BB4_4 Depth=1166; CHECK-NEXT: @ => This Inner Loop Header: Depth=2167; CHECK-NEXT: vldrh.u16 q0, [r4], #16168; CHECK-NEXT: vldrh.u16 q1, [r5], #16169; CHECK-NEXT: rsb.w r7, r12, #0170; CHECK-NEXT: vmullb.s16 q2, q1, q0171; CHECK-NEXT: vmullt.s16 q0, q1, q0172; CHECK-NEXT: vshl.s32 q2, r7173; CHECK-NEXT: vshl.s32 q0, r7174; CHECK-NEXT: vaddva.u32 r6, q2175; CHECK-NEXT: vaddva.u32 r6, q0176; CHECK-NEXT: le lr, .LBB4_8177; CHECK-NEXT: @ %bb.9: @ %middle.block178; CHECK-NEXT: @ in Loop: Header=BB4_4 Depth=1179; CHECK-NEXT: mov r5, r2180; CHECK-NEXT: cmp r0, r1181; CHECK-NEXT: beq .LBB4_3182; CHECK-NEXT: .LBB4_10: @ %vec.epilog.ph183; CHECK-NEXT: @ in Loop: Header=BB4_4 Depth=1184; CHECK-NEXT: ldr r0, [sp] @ 4-byte Reload185; CHECK-NEXT: add.w r4, r1, r8186; CHECK-NEXT: sub.w r7, r9, r1187; CHECK-NEXT: add.w r2, r0, r1, lsl #1188; CHECK-NEXT: add.w r4, r0, r4, lsl #1189; CHECK-NEXT: mov r1, r5190; CHECK-NEXT: mov r5, r1191; CHECK-NEXT: dlstp.32 lr, r7192; CHECK-NEXT: .LBB4_11: @ %vec.epilog.vector.body193; CHECK-NEXT: @ Parent Loop BB4_4 Depth=1194; CHECK-NEXT: @ => This Inner Loop Header: Depth=2195; CHECK-NEXT: rsb.w r0, r12, #0196; CHECK-NEXT: vldrh.s32 q0, [r2], #8197; CHECK-NEXT: vldrh.s32 q1, [r4], #8198; CHECK-NEXT: vmul.i32 q0, q1, q0199; CHECK-NEXT: vshl.s32 q0, r0200; CHECK-NEXT: vaddva.u32 r6, q0201; CHECK-NEXT: letp lr, .LBB4_11202; CHECK-NEXT: b .LBB4_3203; CHECK-NEXT: .LBB4_12: @ %for.end17204; CHECK-NEXT: add sp, #8205; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}206entry:207 %conv = sext i16 %Ls to i32208 %cmp31 = icmp sgt i16 %Ls, 0209 br i1 %cmp31, label %for.body.lr.ph, label %for.end17210 211for.body.lr.ph: ; preds = %entry212 %conv2 = sext i16 %DS to i32213 %conv1027 = zext i16 %S to i32214 %broadcast.splatinsert = insertelement <8 x i32> poison, i32 %conv1027, i64 0215 %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> poison, <8 x i32> zeroinitializer216 %broadcast.splatinsert40 = insertelement <4 x i32> poison, i32 %conv1027, i64 0217 %broadcast.splat41 = shufflevector <4 x i32> %broadcast.splatinsert40, <4 x i32> poison, <4 x i32> zeroinitializer218 br label %for.body219 220for.body: ; preds = %for.body.lr.ph, %for.end221 %lag.032 = phi i32 [ 0, %for.body.lr.ph ], [ %inc16, %for.end ]222 %0 = sub i32 %conv2, %lag.032223 %cmp428 = icmp slt i32 %lag.032, %conv2224 br i1 %cmp428, label %vector.main.loop.iter.check, label %for.end225 226vector.main.loop.iter.check: ; preds = %for.body227 %min.iters.check = icmp ult i32 %0, 8228 br i1 %min.iters.check, label %vec.epilog.ph, label %vector.ph229 230vector.ph: ; preds = %vector.main.loop.iter.check231 %n.vec = and i32 %0, -8232 br label %vector.body233 234vector.body: ; preds = %vector.body, %vector.ph235 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]236 %vec.phi = phi i32 [ 0, %vector.ph ], [ %9, %vector.body ]237 %1 = getelementptr inbounds i16, ptr %ID, i32 %index238 %wide.load = load <8 x i16>, ptr %1, align 2239 %2 = sext <8 x i16> %wide.load to <8 x i32>240 %3 = add nuw nsw i32 %index, %lag.032241 %4 = getelementptr inbounds i16, ptr %ID, i32 %3242 %wide.load34 = load <8 x i16>, ptr %4, align 2243 %5 = sext <8 x i16> %wide.load34 to <8 x i32>244 %6 = mul nsw <8 x i32> %5, %2245 %7 = ashr <8 x i32> %6, %broadcast.splat246 %8 = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %7)247 %9 = add i32 %8, %vec.phi248 %index.next = add nuw i32 %index, 8249 %10 = icmp eq i32 %index.next, %n.vec250 br i1 %10, label %middle.block, label %vector.body251 252middle.block: ; preds = %vector.body253 %cmp.n = icmp eq i32 %0, %n.vec254 br i1 %cmp.n, label %for.end, label %vec.epilog.ph255 256vec.epilog.ph: ; preds = %middle.block, %vector.main.loop.iter.check257 %bc.merge.rdx = phi i32 [ 0, %vector.main.loop.iter.check ], [ %9, %middle.block ]258 %vec.epilog.resume.val = phi i32 [ 0, %vector.main.loop.iter.check ], [ %n.vec, %middle.block ]259 %n.rnd.up = add i32 %0, 3260 %n.vec36 = and i32 %n.rnd.up, -4261 br label %vec.epilog.vector.body262 263vec.epilog.vector.body: ; preds = %vec.epilog.vector.body, %vec.epilog.ph264 %index37 = phi i32 [ %vec.epilog.resume.val, %vec.epilog.ph ], [ %index.next42, %vec.epilog.vector.body ]265 %vec.phi38 = phi i32 [ %bc.merge.rdx, %vec.epilog.ph ], [ %20, %vec.epilog.vector.body ]266 %active.lane.mask = tail call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index37, i32 %0)267 %11 = getelementptr inbounds i16, ptr %ID, i32 %index37268 %wide.masked.load = tail call <4 x i16> @llvm.masked.load.v4i16.p0(ptr %11, i32 2, <4 x i1> %active.lane.mask, <4 x i16> poison)269 %12 = sext <4 x i16> %wide.masked.load to <4 x i32>270 %13 = add nuw nsw i32 %index37, %lag.032271 %14 = getelementptr inbounds i16, ptr %ID, i32 %13272 %wide.masked.load39 = tail call <4 x i16> @llvm.masked.load.v4i16.p0(ptr %14, i32 2, <4 x i1> %active.lane.mask, <4 x i16> poison)273 %15 = sext <4 x i16> %wide.masked.load39 to <4 x i32>274 %16 = mul nsw <4 x i32> %15, %12275 %17 = ashr <4 x i32> %16, %broadcast.splat41276 %18 = select <4 x i1> %active.lane.mask, <4 x i32> %17, <4 x i32> zeroinitializer277 %19 = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %18)278 %20 = add i32 %19, %vec.phi38279 %index.next42 = add i32 %index37, 4280 %21 = icmp eq i32 %index.next42, %n.vec36281 br i1 %21, label %for.end, label %vec.epilog.vector.body282 283for.end: ; preds = %vec.epilog.vector.body, %middle.block, %for.body284 %Accumulator.0.lcssa = phi i32 [ 0, %for.body ], [ %9, %middle.block ], [ %20, %vec.epilog.vector.body ]285 %22 = lshr i32 %Accumulator.0.lcssa, 16286 %conv13 = trunc i32 %22 to i16287 %arrayidx14 = getelementptr inbounds i16, ptr %ACD, i32 %lag.032288 store i16 %conv13, ptr %arrayidx14, align 2289 %inc16 = add nuw nsw i32 %lag.032, 1290 %exitcond33.not = icmp eq i32 %inc16, %conv291 br i1 %exitcond33.not, label %for.end17, label %for.body292 293for.end17: ; preds = %for.end, %entry294 ret void295}296 297declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %sh)298declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %sh)299declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)300declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)301declare <4 x i16> @llvm.masked.load.v4i16.p0(ptr nocapture, i32 immarg, <4 x i1>, <4 x i16>)302declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)303