1040 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK3 4define arm_aapcs_vfpcc <2 x i32> @vmulhs_v2i32(<2 x i32> %s0, <2 x i32> %s1) {5; CHECK-LABEL: vmulhs_v2i32:6; CHECK: @ %bb.0: @ %entry7; CHECK-NEXT: vmullb.s32 q2, q0, q18; CHECK-NEXT: vmov r0, s119; CHECK-NEXT: vmov r1, s910; CHECK-NEXT: vmov q0[2], q0[0], r1, r011; CHECK-NEXT: asrs r0, r0, #3112; CHECK-NEXT: asrs r1, r1, #3113; CHECK-NEXT: vmov q0[3], q0[1], r1, r014; CHECK-NEXT: bx lr15entry:16 %s0s = sext <2 x i32> %s0 to <2 x i64>17 %s1s = sext <2 x i32> %s1 to <2 x i64>18 %m = mul <2 x i64> %s0s, %s1s19 %s = ashr <2 x i64> %m, <i64 32, i64 32>20 %s2 = trunc <2 x i64> %s to <2 x i32>21 ret <2 x i32> %s222}23 24define arm_aapcs_vfpcc <2 x i32> @vmulhu_v2i32(<2 x i32> %s0, <2 x i32> %s1) {25; CHECK-LABEL: vmulhu_v2i32:26; CHECK: @ %bb.0: @ %entry27; CHECK-NEXT: vmullb.u32 q2, q0, q128; CHECK-NEXT: vldr s1, .LCPI1_029; CHECK-NEXT: vmov.f32 s0, s930; CHECK-NEXT: vmov.f32 s2, s1131; CHECK-NEXT: vmov.f32 s3, s132; CHECK-NEXT: bx lr33; CHECK-NEXT: .p2align 234; CHECK-NEXT: @ %bb.1:35; CHECK-NEXT: .LCPI1_0:36; CHECK-NEXT: .long 0x00000000 @ float 037entry:38 %s0s = zext <2 x i32> %s0 to <2 x i64>39 %s1s = zext <2 x i32> %s1 to <2 x i64>40 %m = mul <2 x i64> %s0s, %s1s41 %s = lshr <2 x i64> %m, <i64 32, i64 32>42 %s2 = trunc <2 x i64> %s to <2 x i32>43 ret <2 x i32> %s244}45 46define arm_aapcs_vfpcc <4 x i32> @vmulhs_v4i32(<4 x i32> %s0, <4 x i32> %s1) {47; CHECK-LABEL: vmulhs_v4i32:48; CHECK: @ %bb.0: @ %entry49; CHECK-NEXT: vmulh.s32 q0, q0, q150; CHECK-NEXT: bx lr51entry:52 %s0s = sext <4 x i32> %s0 to <4 x i64>53 %s1s = sext <4 x i32> %s1 to <4 x i64>54 %m = mul <4 x i64> %s0s, %s1s55 %s = ashr <4 x i64> %m, <i64 32, i64 32, i64 32, i64 32>56 %s2 = trunc <4 x i64> %s to <4 x i32>57 ret <4 x i32> %s258}59 60define arm_aapcs_vfpcc <4 x i32> @vmulhu_v4i32(<4 x i32> %s0, <4 x i32> %s1) {61; CHECK-LABEL: vmulhu_v4i32:62; CHECK: @ %bb.0: @ %entry63; CHECK-NEXT: vmulh.u32 q0, q0, q164; CHECK-NEXT: bx lr65entry:66 %s0s = zext <4 x i32> %s0 to <4 x i64>67 %s1s = zext <4 x i32> %s1 to <4 x i64>68 %m = mul <4 x i64> %s0s, %s1s69 %s = lshr <4 x i64> %m, <i64 32, i64 32, i64 32, i64 32>70 %s2 = trunc <4 x i64> %s to <4 x i32>71 ret <4 x i32> %s272}73 74define arm_aapcs_vfpcc <4 x i16> @vmulhs_v4i16(<4 x i16> %s0, <4 x i16> %s1) {75; CHECK-LABEL: vmulhs_v4i16:76; CHECK: @ %bb.0: @ %entry77; CHECK-NEXT: vmullb.s16 q0, q0, q178; CHECK-NEXT: vshr.s32 q0, q0, #1679; CHECK-NEXT: bx lr80entry:81 %s0s = sext <4 x i16> %s0 to <4 x i32>82 %s1s = sext <4 x i16> %s1 to <4 x i32>83 %m = mul <4 x i32> %s0s, %s1s84 %s = ashr <4 x i32> %m, <i32 16, i32 16, i32 16, i32 16>85 %s2 = trunc <4 x i32> %s to <4 x i16>86 ret <4 x i16> %s287}88 89define arm_aapcs_vfpcc <4 x i16> @vmulhu_v4i16(<4 x i16> %s0, <4 x i16> %s1) {90; CHECK-LABEL: vmulhu_v4i16:91; CHECK: @ %bb.0: @ %entry92; CHECK-NEXT: vmullb.u16 q0, q0, q193; CHECK-NEXT: vshr.u32 q0, q0, #1694; CHECK-NEXT: bx lr95entry:96 %s0s = zext <4 x i16> %s0 to <4 x i32>97 %s1s = zext <4 x i16> %s1 to <4 x i32>98 %m = mul <4 x i32> %s0s, %s1s99 %s = lshr <4 x i32> %m, <i32 16, i32 16, i32 16, i32 16>100 %s2 = trunc <4 x i32> %s to <4 x i16>101 ret <4 x i16> %s2102}103 104define arm_aapcs_vfpcc <8 x i16> @vmulhs_v8i16(<8 x i16> %s0, <8 x i16> %s1) {105; CHECK-LABEL: vmulhs_v8i16:106; CHECK: @ %bb.0: @ %entry107; CHECK-NEXT: vmulh.s16 q0, q0, q1108; CHECK-NEXT: bx lr109entry:110 %s0s = sext <8 x i16> %s0 to <8 x i32>111 %s1s = sext <8 x i16> %s1 to <8 x i32>112 %m = mul <8 x i32> %s0s, %s1s113 %s = ashr <8 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>114 %s2 = trunc <8 x i32> %s to <8 x i16>115 ret <8 x i16> %s2116}117 118define arm_aapcs_vfpcc <8 x i16> @vmulhu_v8i16(<8 x i16> %s0, <8 x i16> %s1) {119; CHECK-LABEL: vmulhu_v8i16:120; CHECK: @ %bb.0: @ %entry121; CHECK-NEXT: vmulh.u16 q0, q0, q1122; CHECK-NEXT: bx lr123entry:124 %s0s = zext <8 x i16> %s0 to <8 x i32>125 %s1s = zext <8 x i16> %s1 to <8 x i32>126 %m = mul <8 x i32> %s0s, %s1s127 %s = lshr <8 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>128 %s2 = trunc <8 x i32> %s to <8 x i16>129 ret <8 x i16> %s2130}131 132define arm_aapcs_vfpcc <4 x i8> @vmulhs_v4i8(<4 x i8> %s0, <4 x i8> %s1) {133; CHECK-LABEL: vmulhs_v4i8:134; CHECK: @ %bb.0: @ %entry135; CHECK-NEXT: vmovlb.s8 q1, q1136; CHECK-NEXT: vmovlb.s8 q0, q0137; CHECK-NEXT: vmovlb.s16 q1, q1138; CHECK-NEXT: vmovlb.s16 q0, q0139; CHECK-NEXT: vmul.i32 q0, q0, q1140; CHECK-NEXT: vshr.s32 q0, q0, #8141; CHECK-NEXT: bx lr142entry:143 %s0s = sext <4 x i8> %s0 to <4 x i16>144 %s1s = sext <4 x i8> %s1 to <4 x i16>145 %m = mul <4 x i16> %s0s, %s1s146 %s = ashr <4 x i16> %m, <i16 8, i16 8, i16 8, i16 8>147 %s2 = trunc <4 x i16> %s to <4 x i8>148 ret <4 x i8> %s2149}150 151define arm_aapcs_vfpcc <4 x i8> @vmulhu_v4i8(<4 x i8> %s0, <4 x i8> %s1) {152; CHECK-LABEL: vmulhu_v4i8:153; CHECK: @ %bb.0: @ %entry154; CHECK-NEXT: vmov.i32 q2, #0xff155; CHECK-NEXT: vand q1, q1, q2156; CHECK-NEXT: vand q0, q0, q2157; CHECK-NEXT: vmul.i32 q0, q0, q1158; CHECK-NEXT: vshr.u32 q0, q0, #8159; CHECK-NEXT: bx lr160entry:161 %s0s = zext <4 x i8> %s0 to <4 x i16>162 %s1s = zext <4 x i8> %s1 to <4 x i16>163 %m = mul <4 x i16> %s0s, %s1s164 %s = lshr <4 x i16> %m, <i16 8, i16 8, i16 8, i16 8>165 %s2 = trunc <4 x i16> %s to <4 x i8>166 ret <4 x i8> %s2167}168 169define arm_aapcs_vfpcc <8 x i8> @vmulhs_v8i8(<8 x i8> %s0, <8 x i8> %s1) {170; CHECK-LABEL: vmulhs_v8i8:171; CHECK: @ %bb.0: @ %entry172; CHECK-NEXT: vmullb.s8 q0, q0, q1173; CHECK-NEXT: vshr.s16 q0, q0, #8174; CHECK-NEXT: bx lr175entry:176 %s0s = sext <8 x i8> %s0 to <8 x i16>177 %s1s = sext <8 x i8> %s1 to <8 x i16>178 %m = mul <8 x i16> %s0s, %s1s179 %s = ashr <8 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>180 %s2 = trunc <8 x i16> %s to <8 x i8>181 ret <8 x i8> %s2182}183 184define arm_aapcs_vfpcc <8 x i8> @vmulhu_v8i8(<8 x i8> %s0, <8 x i8> %s1) {185; CHECK-LABEL: vmulhu_v8i8:186; CHECK: @ %bb.0: @ %entry187; CHECK-NEXT: vmullb.u8 q0, q0, q1188; CHECK-NEXT: vshr.u16 q0, q0, #8189; CHECK-NEXT: bx lr190entry:191 %s0s = zext <8 x i8> %s0 to <8 x i16>192 %s1s = zext <8 x i8> %s1 to <8 x i16>193 %m = mul <8 x i16> %s0s, %s1s194 %s = lshr <8 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>195 %s2 = trunc <8 x i16> %s to <8 x i8>196 ret <8 x i8> %s2197}198 199define arm_aapcs_vfpcc <16 x i8> @vmulhs_v16i8(<16 x i8> %s0, <16 x i8> %s1) {200; CHECK-LABEL: vmulhs_v16i8:201; CHECK: @ %bb.0: @ %entry202; CHECK-NEXT: vmulh.s8 q0, q0, q1203; CHECK-NEXT: bx lr204entry:205 %s0s = sext <16 x i8> %s0 to <16 x i16>206 %s1s = sext <16 x i8> %s1 to <16 x i16>207 %m = mul <16 x i16> %s0s, %s1s208 %s = ashr <16 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>209 %s2 = trunc <16 x i16> %s to <16 x i8>210 ret <16 x i8> %s2211}212 213define arm_aapcs_vfpcc <16 x i8> @vmulhu_v16i8(<16 x i8> %s0, <16 x i8> %s1) {214; CHECK-LABEL: vmulhu_v16i8:215; CHECK: @ %bb.0: @ %entry216; CHECK-NEXT: vmulh.u8 q0, q0, q1217; CHECK-NEXT: bx lr218entry:219 %s0s = zext <16 x i8> %s0 to <16 x i16>220 %s1s = zext <16 x i8> %s1 to <16 x i16>221 %m = mul <16 x i16> %s0s, %s1s222 %s = lshr <16 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>223 %s2 = trunc <16 x i16> %s to <16 x i8>224 ret <16 x i8> %s2225}226 227define void @vmulh_s8(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) {228; CHECK-LABEL: vmulh_s8:229; CHECK: @ %bb.0: @ %entry230; CHECK-NEXT: .save {r7, lr}231; CHECK-NEXT: push {r7, lr}232; CHECK-NEXT: mov.w lr, #64233; CHECK-NEXT: .LBB14_1: @ %vector.body234; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1235; CHECK-NEXT: vldrb.u8 q0, [r0], #16236; CHECK-NEXT: vldrb.u8 q1, [r1], #16237; CHECK-NEXT: vmulh.s8 q0, q1, q0238; CHECK-NEXT: vstrb.8 q0, [r2], #16239; CHECK-NEXT: le lr, .LBB14_1240; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup241; CHECK-NEXT: pop {r7, pc}242entry:243 br label %vector.body244 245vector.body: ; preds = %vector.body, %entry246 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]247 %0 = getelementptr inbounds i8, ptr %x, i32 %index248 %wide.load = load <16 x i8>, ptr %0, align 1249 %1 = sext <16 x i8> %wide.load to <16 x i16>250 %2 = getelementptr inbounds i8, ptr %y, i32 %index251 %wide.load17 = load <16 x i8>, ptr %2, align 1252 %3 = sext <16 x i8> %wide.load17 to <16 x i16>253 %4 = mul nsw <16 x i16> %3, %1254 %5 = lshr <16 x i16> %4, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>255 %6 = trunc <16 x i16> %5 to <16 x i8>256 %7 = getelementptr inbounds i8, ptr %z, i32 %index257 store <16 x i8> %6, ptr %7, align 1258 %index.next = add i32 %index, 16259 %8 = icmp eq i32 %index.next, 1024260 br i1 %8, label %for.cond.cleanup, label %vector.body261 262for.cond.cleanup: ; preds = %vector.body263 ret void264}265 266define void @vmulh_s16(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) {267; CHECK-LABEL: vmulh_s16:268; CHECK: @ %bb.0: @ %entry269; CHECK-NEXT: .save {r7, lr}270; CHECK-NEXT: push {r7, lr}271; CHECK-NEXT: mov.w lr, #128272; CHECK-NEXT: .LBB15_1: @ %vector.body273; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1274; CHECK-NEXT: vldrh.u16 q0, [r0], #16275; CHECK-NEXT: vldrh.u16 q1, [r1], #16276; CHECK-NEXT: vmulh.s16 q0, q1, q0277; CHECK-NEXT: vstrb.8 q0, [r2], #16278; CHECK-NEXT: le lr, .LBB15_1279; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup280; CHECK-NEXT: pop {r7, pc}281entry:282 br label %vector.body283 284vector.body: ; preds = %vector.body, %entry285 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]286 %0 = getelementptr inbounds i16, ptr %x, i32 %index287 %wide.load = load <8 x i16>, ptr %0, align 2288 %1 = sext <8 x i16> %wide.load to <8 x i32>289 %2 = getelementptr inbounds i16, ptr %y, i32 %index290 %wide.load17 = load <8 x i16>, ptr %2, align 2291 %3 = sext <8 x i16> %wide.load17 to <8 x i32>292 %4 = mul nsw <8 x i32> %3, %1293 %5 = lshr <8 x i32> %4, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>294 %6 = trunc <8 x i32> %5 to <8 x i16>295 %7 = getelementptr inbounds i16, ptr %z, i32 %index296 store <8 x i16> %6, ptr %7, align 2297 %index.next = add i32 %index, 8298 %8 = icmp eq i32 %index.next, 1024299 br i1 %8, label %for.cond.cleanup, label %vector.body300 301for.cond.cleanup: ; preds = %vector.body302 ret void303}304 305define void @vmulh_s32(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) {306; CHECK-LABEL: vmulh_s32:307; CHECK: @ %bb.0: @ %entry308; CHECK-NEXT: .save {r7, lr}309; CHECK-NEXT: push {r7, lr}310; CHECK-NEXT: mov.w lr, #256311; CHECK-NEXT: .LBB16_1: @ %vector.body312; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1313; CHECK-NEXT: vldrw.u32 q0, [r0], #16314; CHECK-NEXT: vldrw.u32 q1, [r1], #16315; CHECK-NEXT: vmulh.s32 q0, q1, q0316; CHECK-NEXT: vstrb.8 q0, [r2], #16317; CHECK-NEXT: le lr, .LBB16_1318; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup319; CHECK-NEXT: pop {r7, pc}320entry:321 br label %vector.body322 323vector.body: ; preds = %vector.body, %entry324 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]325 %0 = getelementptr inbounds i32, ptr %x, i32 %index326 %wide.load = load <4 x i32>, ptr %0, align 4327 %1 = sext <4 x i32> %wide.load to <4 x i64>328 %2 = getelementptr inbounds i32, ptr %y, i32 %index329 %wide.load17 = load <4 x i32>, ptr %2, align 4330 %3 = sext <4 x i32> %wide.load17 to <4 x i64>331 %4 = mul nsw <4 x i64> %3, %1332 %5 = lshr <4 x i64> %4, <i64 32, i64 32, i64 32, i64 32>333 %6 = trunc <4 x i64> %5 to <4 x i32>334 %7 = getelementptr inbounds i32, ptr %z, i32 %index335 store <4 x i32> %6, ptr %7, align 4336 %index.next = add i32 %index, 4337 %8 = icmp eq i32 %index.next, 1024338 br i1 %8, label %for.cond.cleanup, label %vector.body339 340for.cond.cleanup: ; preds = %vector.body341 ret void342}343 344define void @vmulh_u8(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) {345; CHECK-LABEL: vmulh_u8:346; CHECK: @ %bb.0: @ %entry347; CHECK-NEXT: .save {r7, lr}348; CHECK-NEXT: push {r7, lr}349; CHECK-NEXT: mov.w lr, #64350; CHECK-NEXT: .LBB17_1: @ %vector.body351; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1352; CHECK-NEXT: vldrb.u8 q0, [r0], #16353; CHECK-NEXT: vldrb.u8 q1, [r1], #16354; CHECK-NEXT: vmulh.u8 q0, q1, q0355; CHECK-NEXT: vstrb.8 q0, [r2], #16356; CHECK-NEXT: le lr, .LBB17_1357; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup358; CHECK-NEXT: pop {r7, pc}359entry:360 br label %vector.body361 362vector.body: ; preds = %vector.body, %entry363 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]364 %0 = getelementptr inbounds i8, ptr %x, i32 %index365 %wide.load = load <16 x i8>, ptr %0, align 1366 %1 = zext <16 x i8> %wide.load to <16 x i16>367 %2 = getelementptr inbounds i8, ptr %y, i32 %index368 %wide.load17 = load <16 x i8>, ptr %2, align 1369 %3 = zext <16 x i8> %wide.load17 to <16 x i16>370 %4 = mul nuw <16 x i16> %3, %1371 %5 = lshr <16 x i16> %4, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>372 %6 = trunc <16 x i16> %5 to <16 x i8>373 %7 = getelementptr inbounds i8, ptr %z, i32 %index374 store <16 x i8> %6, ptr %7, align 1375 %index.next = add i32 %index, 16376 %8 = icmp eq i32 %index.next, 1024377 br i1 %8, label %for.cond.cleanup, label %vector.body378 379for.cond.cleanup: ; preds = %vector.body380 ret void381}382 383define void @vmulh_u16(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) {384; CHECK-LABEL: vmulh_u16:385; CHECK: @ %bb.0: @ %entry386; CHECK-NEXT: .save {r7, lr}387; CHECK-NEXT: push {r7, lr}388; CHECK-NEXT: mov.w lr, #128389; CHECK-NEXT: .LBB18_1: @ %vector.body390; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1391; CHECK-NEXT: vldrh.u16 q0, [r0], #16392; CHECK-NEXT: vldrh.u16 q1, [r1], #16393; CHECK-NEXT: vmulh.u16 q0, q1, q0394; CHECK-NEXT: vstrb.8 q0, [r2], #16395; CHECK-NEXT: le lr, .LBB18_1396; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup397; CHECK-NEXT: pop {r7, pc}398entry:399 br label %vector.body400 401vector.body: ; preds = %vector.body, %entry402 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]403 %0 = getelementptr inbounds i16, ptr %x, i32 %index404 %wide.load = load <8 x i16>, ptr %0, align 2405 %1 = zext <8 x i16> %wide.load to <8 x i32>406 %2 = getelementptr inbounds i16, ptr %y, i32 %index407 %wide.load17 = load <8 x i16>, ptr %2, align 2408 %3 = zext <8 x i16> %wide.load17 to <8 x i32>409 %4 = mul nuw <8 x i32> %3, %1410 %5 = lshr <8 x i32> %4, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>411 %6 = trunc <8 x i32> %5 to <8 x i16>412 %7 = getelementptr inbounds i16, ptr %z, i32 %index413 store <8 x i16> %6, ptr %7, align 2414 %index.next = add i32 %index, 8415 %8 = icmp eq i32 %index.next, 1024416 br i1 %8, label %for.cond.cleanup, label %vector.body417 418for.cond.cleanup: ; preds = %vector.body419 ret void420}421 422define void @vmulh_u32(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) {423; CHECK-LABEL: vmulh_u32:424; CHECK: @ %bb.0: @ %entry425; CHECK-NEXT: .save {r7, lr}426; CHECK-NEXT: push {r7, lr}427; CHECK-NEXT: mov.w lr, #256428; CHECK-NEXT: .LBB19_1: @ %vector.body429; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1430; CHECK-NEXT: vldrw.u32 q0, [r0], #16431; CHECK-NEXT: vldrw.u32 q1, [r1], #16432; CHECK-NEXT: vmulh.u32 q0, q1, q0433; CHECK-NEXT: vstrb.8 q0, [r2], #16434; CHECK-NEXT: le lr, .LBB19_1435; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup436; CHECK-NEXT: pop {r7, pc}437entry:438 br label %vector.body439 440vector.body: ; preds = %vector.body, %entry441 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]442 %0 = getelementptr inbounds i32, ptr %x, i32 %index443 %wide.load = load <4 x i32>, ptr %0, align 4444 %1 = zext <4 x i32> %wide.load to <4 x i64>445 %2 = getelementptr inbounds i32, ptr %y, i32 %index446 %wide.load17 = load <4 x i32>, ptr %2, align 4447 %3 = zext <4 x i32> %wide.load17 to <4 x i64>448 %4 = mul nuw <4 x i64> %3, %1449 %5 = lshr <4 x i64> %4, <i64 32, i64 32, i64 32, i64 32>450 %6 = trunc <4 x i64> %5 to <4 x i32>451 %7 = getelementptr inbounds i32, ptr %z, i32 %index452 store <4 x i32> %6, ptr %7, align 4453 %index.next = add i32 %index, 4454 %8 = icmp eq i32 %index.next, 1024455 br i1 %8, label %for.cond.cleanup, label %vector.body456 457for.cond.cleanup: ; preds = %vector.body458 ret void459}460 461 462define void @vmulh_s32_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) {463; CHECK-LABEL: vmulh_s32_pred:464; CHECK: @ %bb.0: @ %entry465; CHECK-NEXT: .save {r7, lr}466; CHECK-NEXT: push {r7, lr}467; CHECK-NEXT: cmp r3, #1468; CHECK-NEXT: it lt469; CHECK-NEXT: poplt {r7, pc}470; CHECK-NEXT: .LBB20_1: @ %vector.ph471; CHECK-NEXT: dlstp.32 lr, r3472; CHECK-NEXT: .LBB20_2: @ %vector.body473; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1474; CHECK-NEXT: vldrw.u32 q0, [r1], #16475; CHECK-NEXT: vldrw.u32 q1, [r2], #16476; CHECK-NEXT: vmulh.s32 q0, q1, q0477; CHECK-NEXT: vstrw.32 q0, [r0], #16478; CHECK-NEXT: letp lr, .LBB20_2479; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup480; CHECK-NEXT: pop {r7, pc}481entry:482 %cmp10 = icmp sgt i32 %n, 0483 br i1 %cmp10, label %vector.ph, label %for.cond.cleanup484 485vector.ph: ; preds = %entry486 %n.rnd.up = add i32 %n, 3487 %n.vec = and i32 %n.rnd.up, -4488 br label %vector.body489 490vector.body: ; preds = %vector.body, %vector.ph491 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]492 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %n)493 %0 = getelementptr inbounds i32, ptr %x, i32 %index494 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %0, i32 4, <4 x i1> %active.lane.mask, <4 x i32> poison)495 %1 = sext <4 x i32> %wide.masked.load to <4 x i64>496 %2 = getelementptr inbounds i32, ptr %y, i32 %index497 %wide.masked.load12 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %2, i32 4, <4 x i1> %active.lane.mask, <4 x i32> poison)498 %3 = sext <4 x i32> %wide.masked.load12 to <4 x i64>499 %4 = mul nsw <4 x i64> %3, %1500 %5 = lshr <4 x i64> %4, <i64 32, i64 32, i64 32, i64 32>501 %6 = trunc <4 x i64> %5 to <4 x i32>502 %7 = getelementptr inbounds i32, ptr %d, i32 %index503 call void @llvm.masked.store.v4i32.p0(<4 x i32> %6, ptr %7, i32 4, <4 x i1> %active.lane.mask)504 %index.next = add i32 %index, 4505 %8 = icmp eq i32 %index.next, %n.vec506 br i1 %8, label %for.cond.cleanup, label %vector.body507 508for.cond.cleanup: ; preds = %vector.body, %entry509 ret void510}511 512define void @vmulh_u32_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) {513; CHECK-LABEL: vmulh_u32_pred:514; CHECK: @ %bb.0: @ %entry515; CHECK-NEXT: .save {r7, lr}516; CHECK-NEXT: push {r7, lr}517; CHECK-NEXT: cmp r3, #1518; CHECK-NEXT: it lt519; CHECK-NEXT: poplt {r7, pc}520; CHECK-NEXT: .LBB21_1: @ %vector.ph521; CHECK-NEXT: dlstp.32 lr, r3522; CHECK-NEXT: .LBB21_2: @ %vector.body523; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1524; CHECK-NEXT: vldrw.u32 q0, [r1], #16525; CHECK-NEXT: vldrw.u32 q1, [r2], #16526; CHECK-NEXT: vmulh.u32 q0, q1, q0527; CHECK-NEXT: vstrw.32 q0, [r0], #16528; CHECK-NEXT: letp lr, .LBB21_2529; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup530; CHECK-NEXT: pop {r7, pc}531entry:532 %cmp10 = icmp sgt i32 %n, 0533 br i1 %cmp10, label %vector.ph, label %for.cond.cleanup534 535vector.ph: ; preds = %entry536 %n.rnd.up = add i32 %n, 3537 %n.vec = and i32 %n.rnd.up, -4538 br label %vector.body539 540vector.body: ; preds = %vector.body, %vector.ph541 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]542 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %n)543 %0 = getelementptr inbounds i32, ptr %x, i32 %index544 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %0, i32 4, <4 x i1> %active.lane.mask, <4 x i32> poison)545 %1 = zext <4 x i32> %wide.masked.load to <4 x i64>546 %2 = getelementptr inbounds i32, ptr %y, i32 %index547 %wide.masked.load12 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %2, i32 4, <4 x i1> %active.lane.mask, <4 x i32> poison)548 %3 = zext <4 x i32> %wide.masked.load12 to <4 x i64>549 %4 = mul nuw <4 x i64> %3, %1550 %5 = lshr <4 x i64> %4, <i64 32, i64 32, i64 32, i64 32>551 %6 = trunc <4 x i64> %5 to <4 x i32>552 %7 = getelementptr inbounds i32, ptr %d, i32 %index553 call void @llvm.masked.store.v4i32.p0(<4 x i32> %6, ptr %7, i32 4, <4 x i1> %active.lane.mask)554 %index.next = add i32 %index, 4555 %8 = icmp eq i32 %index.next, %n.vec556 br i1 %8, label %for.cond.cleanup, label %vector.body557 558for.cond.cleanup: ; preds = %vector.body, %entry559 ret void560}561 562define void @vmulh_s16_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) {563; CHECK-LABEL: vmulh_s16_pred:564; CHECK: @ %bb.0: @ %entry565; CHECK-NEXT: .save {r7, lr}566; CHECK-NEXT: push {r7, lr}567; CHECK-NEXT: cmp r3, #1568; CHECK-NEXT: it lt569; CHECK-NEXT: poplt {r7, pc}570; CHECK-NEXT: .LBB22_1: @ %vector.ph571; CHECK-NEXT: dlstp.16 lr, r3572; CHECK-NEXT: .LBB22_2: @ %vector.body573; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1574; CHECK-NEXT: vldrh.u16 q0, [r1], #16575; CHECK-NEXT: vldrh.u16 q1, [r2], #16576; CHECK-NEXT: vmulh.s16 q0, q1, q0577; CHECK-NEXT: vstrh.16 q0, [r0], #16578; CHECK-NEXT: letp lr, .LBB22_2579; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup580; CHECK-NEXT: pop {r7, pc}581entry:582 %cmp10 = icmp sgt i32 %n, 0583 br i1 %cmp10, label %vector.ph, label %for.cond.cleanup584 585vector.ph: ; preds = %entry586 %n.rnd.up = add i32 %n, 7587 %n.vec = and i32 %n.rnd.up, -8588 br label %vector.body589 590vector.body: ; preds = %vector.body, %vector.ph591 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]592 %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %n)593 %0 = getelementptr inbounds i16, ptr %x, i32 %index594 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %0, i32 2, <8 x i1> %active.lane.mask, <8 x i16> poison)595 %1 = sext <8 x i16> %wide.masked.load to <8 x i32>596 %2 = getelementptr inbounds i16, ptr %y, i32 %index597 %wide.masked.load12 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %2, i32 2, <8 x i1> %active.lane.mask, <8 x i16> poison)598 %3 = sext <8 x i16> %wide.masked.load12 to <8 x i32>599 %4 = mul nsw <8 x i32> %3, %1600 %5 = lshr <8 x i32> %4, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>601 %6 = trunc <8 x i32> %5 to <8 x i16>602 %7 = getelementptr inbounds i16, ptr %d, i32 %index603 call void @llvm.masked.store.v8i16.p0(<8 x i16> %6, ptr %7, i32 2, <8 x i1> %active.lane.mask)604 %index.next = add i32 %index, 8605 %8 = icmp eq i32 %index.next, %n.vec606 br i1 %8, label %for.cond.cleanup, label %vector.body607 608for.cond.cleanup: ; preds = %vector.body, %entry609 ret void610}611 612define void @vmulh_u16_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) {613; CHECK-LABEL: vmulh_u16_pred:614; CHECK: @ %bb.0: @ %entry615; CHECK-NEXT: .save {r7, lr}616; CHECK-NEXT: push {r7, lr}617; CHECK-NEXT: cmp r3, #1618; CHECK-NEXT: it lt619; CHECK-NEXT: poplt {r7, pc}620; CHECK-NEXT: .LBB23_1: @ %vector.ph621; CHECK-NEXT: dlstp.16 lr, r3622; CHECK-NEXT: .LBB23_2: @ %vector.body623; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1624; CHECK-NEXT: vldrh.u16 q0, [r1], #16625; CHECK-NEXT: vldrh.u16 q1, [r2], #16626; CHECK-NEXT: vmulh.u16 q0, q1, q0627; CHECK-NEXT: vstrh.16 q0, [r0], #16628; CHECK-NEXT: letp lr, .LBB23_2629; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup630; CHECK-NEXT: pop {r7, pc}631entry:632 %cmp10 = icmp sgt i32 %n, 0633 br i1 %cmp10, label %vector.ph, label %for.cond.cleanup634 635vector.ph: ; preds = %entry636 %n.rnd.up = add i32 %n, 7637 %n.vec = and i32 %n.rnd.up, -8638 br label %vector.body639 640vector.body: ; preds = %vector.body, %vector.ph641 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]642 %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %n)643 %0 = getelementptr inbounds i16, ptr %x, i32 %index644 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %0, i32 2, <8 x i1> %active.lane.mask, <8 x i16> poison)645 %1 = zext <8 x i16> %wide.masked.load to <8 x i32>646 %2 = getelementptr inbounds i16, ptr %y, i32 %index647 %wide.masked.load12 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %2, i32 2, <8 x i1> %active.lane.mask, <8 x i16> poison)648 %3 = zext <8 x i16> %wide.masked.load12 to <8 x i32>649 %4 = mul nuw <8 x i32> %3, %1650 %5 = lshr <8 x i32> %4, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>651 %6 = trunc <8 x i32> %5 to <8 x i16>652 %7 = getelementptr inbounds i16, ptr %d, i32 %index653 call void @llvm.masked.store.v8i16.p0(<8 x i16> %6, ptr %7, i32 2, <8 x i1> %active.lane.mask)654 %index.next = add i32 %index, 8655 %8 = icmp eq i32 %index.next, %n.vec656 br i1 %8, label %for.cond.cleanup, label %vector.body657 658for.cond.cleanup: ; preds = %vector.body, %entry659 ret void660}661 662define void @vmulh_s8_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) {663; CHECK-LABEL: vmulh_s8_pred:664; CHECK: @ %bb.0: @ %entry665; CHECK-NEXT: .save {r7, lr}666; CHECK-NEXT: push {r7, lr}667; CHECK-NEXT: cmp r3, #1668; CHECK-NEXT: it lt669; CHECK-NEXT: poplt {r7, pc}670; CHECK-NEXT: .LBB24_1: @ %vector.ph671; CHECK-NEXT: dlstp.8 lr, r3672; CHECK-NEXT: .LBB24_2: @ %vector.body673; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1674; CHECK-NEXT: vldrb.u8 q0, [r1], #16675; CHECK-NEXT: vldrb.u8 q1, [r2], #16676; CHECK-NEXT: vmulh.s8 q0, q1, q0677; CHECK-NEXT: vstrb.8 q0, [r0], #16678; CHECK-NEXT: letp lr, .LBB24_2679; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup680; CHECK-NEXT: pop {r7, pc}681entry:682 %cmp10 = icmp sgt i32 %n, 0683 br i1 %cmp10, label %vector.ph, label %for.cond.cleanup684 685vector.ph: ; preds = %entry686 %n.rnd.up = add i32 %n, 15687 %n.vec = and i32 %n.rnd.up, -16688 br label %vector.body689 690vector.body: ; preds = %vector.body, %vector.ph691 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]692 %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %n)693 %0 = getelementptr inbounds i8, ptr %x, i32 %index694 %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %0, i32 1, <16 x i1> %active.lane.mask, <16 x i8> poison)695 %1 = sext <16 x i8> %wide.masked.load to <16 x i16>696 %2 = getelementptr inbounds i8, ptr %y, i32 %index697 %wide.masked.load12 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %2, i32 1, <16 x i1> %active.lane.mask, <16 x i8> poison)698 %3 = sext <16 x i8> %wide.masked.load12 to <16 x i16>699 %4 = mul nsw <16 x i16> %3, %1700 %5 = lshr <16 x i16> %4, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>701 %6 = trunc <16 x i16> %5 to <16 x i8>702 %7 = getelementptr inbounds i8, ptr %d, i32 %index703 call void @llvm.masked.store.v16i8.p0(<16 x i8> %6, ptr %7, i32 1, <16 x i1> %active.lane.mask)704 %index.next = add i32 %index, 16705 %8 = icmp eq i32 %index.next, %n.vec706 br i1 %8, label %for.cond.cleanup, label %vector.body707 708for.cond.cleanup: ; preds = %vector.body, %entry709 ret void710}711 712define void @vmulh_u8_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) {713; CHECK-LABEL: vmulh_u8_pred:714; CHECK: @ %bb.0: @ %entry715; CHECK-NEXT: .save {r7, lr}716; CHECK-NEXT: push {r7, lr}717; CHECK-NEXT: cmp r3, #1718; CHECK-NEXT: it lt719; CHECK-NEXT: poplt {r7, pc}720; CHECK-NEXT: .LBB25_1: @ %vector.ph721; CHECK-NEXT: dlstp.8 lr, r3722; CHECK-NEXT: .LBB25_2: @ %vector.body723; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1724; CHECK-NEXT: vldrb.u8 q0, [r1], #16725; CHECK-NEXT: vldrb.u8 q1, [r2], #16726; CHECK-NEXT: vmulh.u8 q0, q1, q0727; CHECK-NEXT: vstrb.8 q0, [r0], #16728; CHECK-NEXT: letp lr, .LBB25_2729; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup730; CHECK-NEXT: pop {r7, pc}731entry:732 %cmp10 = icmp sgt i32 %n, 0733 br i1 %cmp10, label %vector.ph, label %for.cond.cleanup734 735vector.ph: ; preds = %entry736 %n.rnd.up = add i32 %n, 15737 %n.vec = and i32 %n.rnd.up, -16738 br label %vector.body739 740vector.body: ; preds = %vector.body, %vector.ph741 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]742 %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %n)743 %0 = getelementptr inbounds i8, ptr %x, i32 %index744 %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %0, i32 1, <16 x i1> %active.lane.mask, <16 x i8> poison)745 %1 = zext <16 x i8> %wide.masked.load to <16 x i16>746 %2 = getelementptr inbounds i8, ptr %y, i32 %index747 %wide.masked.load12 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %2, i32 1, <16 x i1> %active.lane.mask, <16 x i8> poison)748 %3 = zext <16 x i8> %wide.masked.load12 to <16 x i16>749 %4 = mul nuw <16 x i16> %3, %1750 %5 = lshr <16 x i16> %4, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>751 %6 = trunc <16 x i16> %5 to <16 x i8>752 %7 = getelementptr inbounds i8, ptr %d, i32 %index753 call void @llvm.masked.store.v16i8.p0(<16 x i8> %6, ptr %7, i32 1, <16 x i1> %active.lane.mask)754 %index.next = add i32 %index, 16755 %8 = icmp eq i32 %index.next, %n.vec756 br i1 %8, label %for.cond.cleanup, label %vector.body757 758for.cond.cleanup: ; preds = %vector.body, %entry759 ret void760}761 762 763define arm_aapcs_vfpcc i16 @vmulhs_reduce_v16i8(<16 x i8> %s0, <16 x i8> %s1) {764; CHECK-LABEL: vmulhs_reduce_v16i8:765; CHECK: @ %bb.0: @ %entry766; CHECK-NEXT: vmulh.s8 q0, q0, q1767; CHECK-NEXT: vaddv.s8 r0, q0768; CHECK-NEXT: bx lr769entry:770 %s0s = sext <16 x i8> %s0 to <16 x i16>771 %s1s = sext <16 x i8> %s1 to <16 x i16>772 %m = mul <16 x i16> %s0s, %s1s773 %s = ashr <16 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>774 %result = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s)775 ret i16 %result776}777 778define arm_aapcs_vfpcc i16 @vmulhu_reduce_v16i8(<16 x i8> %s0, <16 x i8> %s1) {779; CHECK-LABEL: vmulhu_reduce_v16i8:780; CHECK: @ %bb.0: @ %entry781; CHECK-NEXT: vmulh.u8 q0, q0, q1782; CHECK-NEXT: vaddv.s8 r0, q0783; CHECK-NEXT: bx lr784entry:785 %s0s = zext <16 x i8> %s0 to <16 x i16>786 %s1s = zext <16 x i8> %s1 to <16 x i16>787 %m = mul <16 x i16> %s0s, %s1s788 %s = ashr <16 x i16> %m, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>789 %result = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s)790 ret i16 %result791}792 793define arm_aapcs_vfpcc <4 x i32> @vmulhs_kb_v4i32(<4 x i32> %s0, <4 x i64> %s1) {794; CHECK-LABEL: vmulhs_kb_v4i32:795; CHECK: @ %bb.0: @ %entry796; CHECK-NEXT: vmov.f32 s4, s2797; CHECK-NEXT: vmov r1, s9798; CHECK-NEXT: vmov r2, s5799; CHECK-NEXT: vmov.f32 s6, s3800; CHECK-NEXT: vmov.f32 s10, s1801; CHECK-NEXT: vmov r0, s4802; CHECK-NEXT: smmul r0, r0, r1803; CHECK-NEXT: vmov r1, s0804; CHECK-NEXT: smmul r1, r1, r2805; CHECK-NEXT: vmov r2, s7806; CHECK-NEXT: vmov q0[2], q0[0], r1, r0807; CHECK-NEXT: vmov r0, s6808; CHECK-NEXT: vmov r1, s11809; CHECK-NEXT: smmul r0, r0, r1810; CHECK-NEXT: vmov r1, s10811; CHECK-NEXT: smmul r1, r1, r2812; CHECK-NEXT: vmov q0[3], q0[1], r1, r0813; CHECK-NEXT: bx lr814entry:815 %s0s = sext <4 x i32> %s0 to <4 x i64>816 %s1s = ashr <4 x i64> %s1, <i64 32, i64 32, i64 32, i64 32>817 %m = mul <4 x i64> %s0s, %s1s818 %s = ashr <4 x i64> %m, <i64 32, i64 32, i64 32, i64 32>819 %s2 = trunc <4 x i64> %s to <4 x i32>820 ret <4 x i32> %s2821}822 823define arm_aapcs_vfpcc <4 x i32> @vmulhu_kb_v4i32(<4 x i32> %s0, <4 x i64> %s1) {824; CHECK-LABEL: vmulhu_kb_v4i32:825; CHECK: @ %bb.0: @ %entry826; CHECK-NEXT: vmov.f32 s4, s2827; CHECK-NEXT: vmov r1, s9828; CHECK-NEXT: vmov r2, s5829; CHECK-NEXT: vmov.f32 s6, s3830; CHECK-NEXT: vmov.f32 s10, s1831; CHECK-NEXT: vmov r0, s4832; CHECK-NEXT: umull r0, r1, r0, r1833; CHECK-NEXT: vmov r0, s0834; CHECK-NEXT: umull r0, r2, r0, r2835; CHECK-NEXT: vmov r0, s6836; CHECK-NEXT: vmov q0[2], q0[0], r2, r1837; CHECK-NEXT: vmov r1, s11838; CHECK-NEXT: vmov r2, s7839; CHECK-NEXT: umull r0, r1, r0, r1840; CHECK-NEXT: vmov r0, s10841; CHECK-NEXT: umull r0, r2, r0, r2842; CHECK-NEXT: vmov q0[3], q0[1], r2, r1843; CHECK-NEXT: bx lr844entry:845 %s0s = zext <4 x i32> %s0 to <4 x i64>846 %s1s = lshr <4 x i64> %s1, <i64 32, i64 32, i64 32, i64 32>847 %m = mul <4 x i64> %s0s, %s1s848 %s = lshr <4 x i64> %m, <i64 32, i64 32, i64 32, i64 32>849 %s2 = trunc <4 x i64> %s to <4 x i32>850 ret <4 x i32> %s2851}852 853define arm_aapcs_vfpcc <4 x i32> @vmulhs_kbc_v4i32(<4 x i32> %s0, <4 x i64> %s1) {854; CHECK-LABEL: vmulhs_kbc_v4i32:855; CHECK: @ %bb.0: @ %entry856; CHECK-NEXT: vmov.f32 s4, s2857; CHECK-NEXT: vmov r1, s9858; CHECK-NEXT: vmov r2, s5859; CHECK-NEXT: vmov.f32 s6, s3860; CHECK-NEXT: vmov.f32 s10, s1861; CHECK-NEXT: vmov r0, s4862; CHECK-NEXT: smmul r0, r1, r0863; CHECK-NEXT: vmov r1, s0864; CHECK-NEXT: smmul r1, r2, r1865; CHECK-NEXT: vmov r2, s7866; CHECK-NEXT: vmov q0[2], q0[0], r1, r0867; CHECK-NEXT: vmov r0, s6868; CHECK-NEXT: vmov r1, s11869; CHECK-NEXT: smmul r0, r1, r0870; CHECK-NEXT: vmov r1, s10871; CHECK-NEXT: smmul r1, r2, r1872; CHECK-NEXT: vmov q0[3], q0[1], r1, r0873; CHECK-NEXT: bx lr874entry:875 %s0s = sext <4 x i32> %s0 to <4 x i64>876 %s1s = ashr <4 x i64> %s1, <i64 32, i64 32, i64 32, i64 32>877 %m = mul <4 x i64> %s1s, %s0s878 %s = ashr <4 x i64> %m, <i64 32, i64 32, i64 32, i64 32>879 %s2 = trunc <4 x i64> %s to <4 x i32>880 ret <4 x i32> %s2881}882 883define arm_aapcs_vfpcc <4 x i32> @vmulhu_kbc_v4i32(<4 x i32> %s0, <4 x i64> %s1) {884; CHECK-LABEL: vmulhu_kbc_v4i32:885; CHECK: @ %bb.0: @ %entry886; CHECK-NEXT: vmov.f32 s4, s2887; CHECK-NEXT: vmov r1, s9888; CHECK-NEXT: vmov r2, s5889; CHECK-NEXT: vmov.f32 s6, s3890; CHECK-NEXT: vmov.f32 s10, s1891; CHECK-NEXT: vmov r0, s4892; CHECK-NEXT: umull r0, r1, r1, r0893; CHECK-NEXT: vmov r0, s0894; CHECK-NEXT: umull r0, r2, r2, r0895; CHECK-NEXT: vmov r0, s6896; CHECK-NEXT: vmov q0[2], q0[0], r2, r1897; CHECK-NEXT: vmov r1, s11898; CHECK-NEXT: vmov r2, s7899; CHECK-NEXT: umull r0, r1, r1, r0900; CHECK-NEXT: vmov r0, s10901; CHECK-NEXT: umull r0, r2, r2, r0902; CHECK-NEXT: vmov q0[3], q0[1], r2, r1903; CHECK-NEXT: bx lr904entry:905 %s0s = zext <4 x i32> %s0 to <4 x i64>906 %s1s = lshr <4 x i64> %s1, <i64 32, i64 32, i64 32, i64 32>907 %m = mul <4 x i64> %s1s, %s0s908 %s = lshr <4 x i64> %m, <i64 32, i64 32, i64 32, i64 32>909 %s2 = trunc <4 x i64> %s to <4 x i32>910 ret <4 x i32> %s2911}912 913define arm_aapcs_vfpcc <8 x i16> @vmulhs_kb_v8i16(<8 x i16> %s0, <8 x i32> %s1) {914; CHECK-LABEL: vmulhs_kb_v8i16:915; CHECK: @ %bb.0: @ %entry916; CHECK-NEXT: .vsave {d8, d9}917; CHECK-NEXT: vpush {d8, d9}918; CHECK-NEXT: vmov.f32 s12, s5919; CHECK-NEXT: vmovlt.s16 q4, q0920; CHECK-NEXT: vmov.f32 s13, s7921; CHECK-NEXT: vmovlb.s16 q0, q0922; CHECK-NEXT: vmov.f32 s5, s6923; CHECK-NEXT: vmov.f32 s14, s9924; CHECK-NEXT: vmov.f32 s15, s11925; CHECK-NEXT: vmov.f32 s6, s8926; CHECK-NEXT: vshr.s32 q3, q3, #16927; CHECK-NEXT: vmov.f32 s7, s10928; CHECK-NEXT: vmul.i32 q3, q4, q3929; CHECK-NEXT: vshr.s32 q1, q1, #16930; CHECK-NEXT: vshr.u32 q3, q3, #16931; CHECK-NEXT: vmul.i32 q0, q0, q1932; CHECK-NEXT: vshr.u32 q0, q0, #16933; CHECK-NEXT: vmovnt.i32 q0, q3934; CHECK-NEXT: vpop {d8, d9}935; CHECK-NEXT: bx lr936entry:937 %s0s = sext <8 x i16> %s0 to <8 x i32>938 %s1s = ashr <8 x i32> %s1, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>939 %m = mul <8 x i32> %s0s, %s1s940 %s = ashr <8 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>941 %s2 = trunc <8 x i32> %s to <8 x i16>942 ret <8 x i16> %s2943}944 945define arm_aapcs_vfpcc <8 x i16> @vmulhu_kb_v8i16(<8 x i16> %s0, <8 x i32> %s1) {946; CHECK-LABEL: vmulhu_kb_v8i16:947; CHECK: @ %bb.0: @ %entry948; CHECK-NEXT: .vsave {d8, d9}949; CHECK-NEXT: vpush {d8, d9}950; CHECK-NEXT: vmov.f32 s12, s5951; CHECK-NEXT: vmovlt.u16 q4, q0952; CHECK-NEXT: vmov.f32 s13, s7953; CHECK-NEXT: vmovlb.u16 q0, q0954; CHECK-NEXT: vmov.f32 s5, s6955; CHECK-NEXT: vmov.f32 s14, s9956; CHECK-NEXT: vmov.f32 s15, s11957; CHECK-NEXT: vmov.f32 s6, s8958; CHECK-NEXT: vshr.u32 q3, q3, #16959; CHECK-NEXT: vmov.f32 s7, s10960; CHECK-NEXT: vmul.i32 q3, q4, q3961; CHECK-NEXT: vshr.u32 q1, q1, #16962; CHECK-NEXT: vshr.u32 q3, q3, #16963; CHECK-NEXT: vmul.i32 q0, q0, q1964; CHECK-NEXT: vshr.u32 q0, q0, #16965; CHECK-NEXT: vmovnt.i32 q0, q3966; CHECK-NEXT: vpop {d8, d9}967; CHECK-NEXT: bx lr968entry:969 %s0s = zext <8 x i16> %s0 to <8 x i32>970 %s1s = lshr <8 x i32> %s1, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>971 %m = mul <8 x i32> %s0s, %s1s972 %s = lshr <8 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>973 %s2 = trunc <8 x i32> %s to <8 x i16>974 ret <8 x i16> %s2975}976 977define arm_aapcs_vfpcc <8 x i16> @vmulhs_kbc_v8i16(<8 x i16> %s0, <8 x i32> %s1) {978; CHECK-LABEL: vmulhs_kbc_v8i16:979; CHECK: @ %bb.0: @ %entry980; CHECK-NEXT: .vsave {d8, d9}981; CHECK-NEXT: vpush {d8, d9}982; CHECK-NEXT: vmov.f32 s12, s5983; CHECK-NEXT: vmovlt.s16 q4, q0984; CHECK-NEXT: vmov.f32 s13, s7985; CHECK-NEXT: vmovlb.s16 q0, q0986; CHECK-NEXT: vmov.f32 s5, s6987; CHECK-NEXT: vmov.f32 s14, s9988; CHECK-NEXT: vmov.f32 s15, s11989; CHECK-NEXT: vmov.f32 s6, s8990; CHECK-NEXT: vshr.s32 q3, q3, #16991; CHECK-NEXT: vmov.f32 s7, s10992; CHECK-NEXT: vmul.i32 q3, q3, q4993; CHECK-NEXT: vshr.s32 q1, q1, #16994; CHECK-NEXT: vshr.u32 q3, q3, #16995; CHECK-NEXT: vmul.i32 q0, q1, q0996; CHECK-NEXT: vshr.u32 q0, q0, #16997; CHECK-NEXT: vmovnt.i32 q0, q3998; CHECK-NEXT: vpop {d8, d9}999; CHECK-NEXT: bx lr1000entry:1001 %s0s = sext <8 x i16> %s0 to <8 x i32>1002 %s1s = ashr <8 x i32> %s1, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1003 %m = mul <8 x i32> %s1s, %s0s1004 %s = ashr <8 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1005 %s2 = trunc <8 x i32> %s to <8 x i16>1006 ret <8 x i16> %s21007}1008 1009define arm_aapcs_vfpcc <8 x i16> @vmulhu_kbc_v8i16(<8 x i16> %s0, <8 x i32> %s1) {1010; CHECK-LABEL: vmulhu_kbc_v8i16:1011; CHECK: @ %bb.0: @ %entry1012; CHECK-NEXT: .vsave {d8, d9}1013; CHECK-NEXT: vpush {d8, d9}1014; CHECK-NEXT: vmov.f32 s12, s51015; CHECK-NEXT: vmovlt.u16 q4, q01016; CHECK-NEXT: vmov.f32 s13, s71017; CHECK-NEXT: vmovlb.u16 q0, q01018; CHECK-NEXT: vmov.f32 s5, s61019; CHECK-NEXT: vmov.f32 s14, s91020; CHECK-NEXT: vmov.f32 s15, s111021; CHECK-NEXT: vmov.f32 s6, s81022; CHECK-NEXT: vshr.u32 q3, q3, #161023; CHECK-NEXT: vmov.f32 s7, s101024; CHECK-NEXT: vmul.i32 q3, q3, q41025; CHECK-NEXT: vshr.u32 q1, q1, #161026; CHECK-NEXT: vshr.u32 q3, q3, #161027; CHECK-NEXT: vmul.i32 q0, q1, q01028; CHECK-NEXT: vshr.u32 q0, q0, #161029; CHECK-NEXT: vmovnt.i32 q0, q31030; CHECK-NEXT: vpop {d8, d9}1031; CHECK-NEXT: bx lr1032entry:1033 %s0s = zext <8 x i16> %s0 to <8 x i32>1034 %s1s = lshr <8 x i32> %s1, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1035 %m = mul <8 x i32> %s1s, %s0s1036 %s = lshr <8 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1037 %s2 = trunc <8 x i32> %s to <8 x i16>1038 ret <8 x i16> %s21039}1040