1061 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2563; RUN: llc -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5124; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5125 6; This test only tests the legal types for a given vector width, as mulh nodes7; do not get generated for non-legal types.8 9target triple = "aarch64-unknown-linux-gnu"10 11;12; SMULH13;14 15; Don't use SVE for 64-bit vectors.16define <8 x i8> @smulh_v8i8(<8 x i8> %op1, <8 x i8> %op2) vscale_range(2,0) #0 {17; CHECK-LABEL: smulh_v8i8:18; CHECK: // %bb.0:19; CHECK-NEXT: ptrue p0.b, vl820; CHECK-NEXT: // kill: def $d0 killed $d0 def $z021; CHECK-NEXT: // kill: def $d1 killed $d1 def $z122; CHECK-NEXT: smulh z0.b, p0/m, z0.b, z1.b23; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z024; CHECK-NEXT: ret25 %1 = sext <8 x i8> %op1 to <8 x i16>26 %2 = sext <8 x i8> %op2 to <8 x i16>27 %mul = mul <8 x i16> %1, %228 %shr = lshr <8 x i16> %mul, splat (i16 8)29 %res = trunc <8 x i16> %shr to <8 x i8>30 ret <8 x i8> %res31}32 33; Don't use SVE for 128-bit vectors.34define <16 x i8> @smulh_v16i8(<16 x i8> %op1, <16 x i8> %op2) vscale_range(2,0) #0 {35; CHECK-LABEL: smulh_v16i8:36; CHECK: // %bb.0:37; CHECK-NEXT: ptrue p0.b, vl1638; CHECK-NEXT: // kill: def $q0 killed $q0 def $z039; CHECK-NEXT: // kill: def $q1 killed $q1 def $z140; CHECK-NEXT: smulh z0.b, p0/m, z0.b, z1.b41; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z042; CHECK-NEXT: ret43 %1 = sext <16 x i8> %op1 to <16 x i16>44 %2 = sext <16 x i8> %op2 to <16 x i16>45 %mul = mul <16 x i16> %1, %246 %shr = lshr <16 x i16> %mul, splat (i16 8)47 %res = trunc <16 x i16> %shr to <16 x i8>48 ret <16 x i8> %res49}50 51define void @smulh_v32i8(ptr %a, ptr %b) vscale_range(2,0) #0 {52; CHECK-LABEL: smulh_v32i8:53; CHECK: // %bb.0:54; CHECK-NEXT: ptrue p0.b, vl3255; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]56; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]57; CHECK-NEXT: smulh z0.b, p0/m, z0.b, z1.b58; CHECK-NEXT: st1b { z0.b }, p0, [x0]59; CHECK-NEXT: ret60 %op1 = load <32 x i8>, ptr %a61 %op2 = load <32 x i8>, ptr %b62 %1 = sext <32 x i8> %op1 to <32 x i16>63 %2 = sext <32 x i8> %op2 to <32 x i16>64 %mul = mul <32 x i16> %1, %265 %shr = lshr <32 x i16> %mul, splat (i16 8)66 %res = trunc <32 x i16> %shr to <32 x i8>67 store <32 x i8> %res, ptr %a68 ret void69}70 71define void @smulh_v64i8(ptr %a, ptr %b) #0 {72; VBITS_GE_256-LABEL: smulh_v64i8:73; VBITS_GE_256: // %bb.0:74; VBITS_GE_256-NEXT: ptrue p0.b, vl3275; VBITS_GE_256-NEXT: mov w8, #32 // =0x2076; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]77; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x1, x8]78; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0]79; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x1]80; VBITS_GE_256-NEXT: smulh z0.b, p0/m, z0.b, z1.b81; VBITS_GE_256-NEXT: smulh z2.b, p0/m, z2.b, z3.b82; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0, x8]83; VBITS_GE_256-NEXT: st1b { z2.b }, p0, [x0]84; VBITS_GE_256-NEXT: ret85;86; VBITS_GE_512-LABEL: smulh_v64i8:87; VBITS_GE_512: // %bb.0:88; VBITS_GE_512-NEXT: ptrue p0.b, vl6489; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]90; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x1]91; VBITS_GE_512-NEXT: smulh z0.b, p0/m, z0.b, z1.b92; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0]93; VBITS_GE_512-NEXT: ret94 %op1 = load <64 x i8>, ptr %a95 %op2 = load <64 x i8>, ptr %b96 %1 = sext <64 x i8> %op1 to <64 x i16>97 %2 = sext <64 x i8> %op2 to <64 x i16>98 %mul = mul <64 x i16> %1, %299 %shr = lshr <64 x i16> %mul, splat (i16 8)100 %res = trunc <64 x i16> %shr to <64 x i8>101 store <64 x i8> %res, ptr %a102 ret void103}104 105define void @smulh_v128i8(ptr %a, ptr %b) vscale_range(8,0) #0 {106; CHECK-LABEL: smulh_v128i8:107; CHECK: // %bb.0:108; CHECK-NEXT: ptrue p0.b, vl128109; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]110; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]111; CHECK-NEXT: smulh z0.b, p0/m, z0.b, z1.b112; CHECK-NEXT: st1b { z0.b }, p0, [x0]113; CHECK-NEXT: ret114 %op1 = load <128 x i8>, ptr %a115 %op2 = load <128 x i8>, ptr %b116 %1 = sext <128 x i8> %op1 to <128 x i16>117 %2 = sext <128 x i8> %op2 to <128 x i16>118 %mul = mul <128 x i16> %1, %2119 %shr = lshr <128 x i16> %mul, splat (i16 8)120 %res = trunc <128 x i16> %shr to <128 x i8>121 store <128 x i8> %res, ptr %a122 ret void123}124 125define void @smulh_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {126; CHECK-LABEL: smulh_v256i8:127; CHECK: // %bb.0:128; CHECK-NEXT: ptrue p0.b, vl256129; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]130; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]131; CHECK-NEXT: smulh z0.b, p0/m, z0.b, z1.b132; CHECK-NEXT: st1b { z0.b }, p0, [x0]133; CHECK-NEXT: ret134 %op1 = load <256 x i8>, ptr %a135 %op2 = load <256 x i8>, ptr %b136 %1 = sext <256 x i8> %op1 to <256 x i16>137 %2 = sext <256 x i8> %op2 to <256 x i16>138 %mul = mul <256 x i16> %1, %2139 %shr = lshr <256 x i16> %mul, splat (i16 8)140 %res = trunc <256 x i16> %shr to <256 x i8>141 store <256 x i8> %res, ptr %a142 ret void143}144 145; Don't use SVE for 64-bit vectors.146define <4 x i16> @smulh_v4i16(<4 x i16> %op1, <4 x i16> %op2) vscale_range(2,0) #0 {147; CHECK-LABEL: smulh_v4i16:148; CHECK: // %bb.0:149; CHECK-NEXT: ptrue p0.h, vl4150; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0151; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1152; CHECK-NEXT: smulh z0.h, p0/m, z0.h, z1.h153; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0154; CHECK-NEXT: ret155 %1 = sext <4 x i16> %op1 to <4 x i32>156 %2 = sext <4 x i16> %op2 to <4 x i32>157 %mul = mul <4 x i32> %1, %2158 %shr = lshr <4 x i32> %mul, splat (i32 16)159 %res = trunc <4 x i32> %shr to <4 x i16>160 ret <4 x i16> %res161}162 163; Don't use SVE for 128-bit vectors.164define <8 x i16> @smulh_v8i16(<8 x i16> %op1, <8 x i16> %op2) vscale_range(2,0) #0 {165; CHECK-LABEL: smulh_v8i16:166; CHECK: // %bb.0:167; CHECK-NEXT: ptrue p0.h, vl8168; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0169; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1170; CHECK-NEXT: smulh z0.h, p0/m, z0.h, z1.h171; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0172; CHECK-NEXT: ret173 %1 = sext <8 x i16> %op1 to <8 x i32>174 %2 = sext <8 x i16> %op2 to <8 x i32>175 %mul = mul <8 x i32> %1, %2176 %shr = lshr <8 x i32> %mul, splat (i32 16)177 %res = trunc <8 x i32> %shr to <8 x i16>178 ret <8 x i16> %res179}180 181define void @smulh_v16i16(ptr %a, ptr %b) vscale_range(2,0) #0 {182; CHECK-LABEL: smulh_v16i16:183; CHECK: // %bb.0:184; CHECK-NEXT: ptrue p0.h, vl16185; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]186; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]187; CHECK-NEXT: smulh z0.h, p0/m, z0.h, z1.h188; CHECK-NEXT: st1h { z0.h }, p0, [x0]189; CHECK-NEXT: ret190 %op1 = load <16 x i16>, ptr %a191 %op2 = load <16 x i16>, ptr %b192 %1 = sext <16 x i16> %op1 to <16 x i32>193 %2 = sext <16 x i16> %op2 to <16 x i32>194 %mul = mul <16 x i32> %1, %2195 %shr = lshr <16 x i32> %mul, splat (i32 16)196 %res = trunc <16 x i32> %shr to <16 x i16>197 store <16 x i16> %res, ptr %a198 ret void199}200 201define void @smulh_v32i16(ptr %a, ptr %b) #0 {202; VBITS_GE_256-LABEL: smulh_v32i16:203; VBITS_GE_256: // %bb.0:204; VBITS_GE_256-NEXT: ptrue p0.h, vl16205; VBITS_GE_256-NEXT: mov x8, #16 // =0x10206; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]207; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1, x8, lsl #1]208; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0]209; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1]210; VBITS_GE_256-NEXT: smulh z0.h, p0/m, z0.h, z1.h211; VBITS_GE_256-NEXT: smulh z2.h, p0/m, z2.h, z3.h212; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]213; VBITS_GE_256-NEXT: st1h { z2.h }, p0, [x0]214; VBITS_GE_256-NEXT: ret215;216; VBITS_GE_512-LABEL: smulh_v32i16:217; VBITS_GE_512: // %bb.0:218; VBITS_GE_512-NEXT: ptrue p0.h, vl32219; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]220; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]221; VBITS_GE_512-NEXT: smulh z0.h, p0/m, z0.h, z1.h222; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]223; VBITS_GE_512-NEXT: ret224 %op1 = load <32 x i16>, ptr %a225 %op2 = load <32 x i16>, ptr %b226 %1 = sext <32 x i16> %op1 to <32 x i32>227 %2 = sext <32 x i16> %op2 to <32 x i32>228 %mul = mul <32 x i32> %1, %2229 %shr = lshr <32 x i32> %mul, splat (i32 16)230 %res = trunc <32 x i32> %shr to <32 x i16>231 store <32 x i16> %res, ptr %a232 ret void233}234 235define void @smulh_v64i16(ptr %a, ptr %b) vscale_range(8,0) #0 {236; CHECK-LABEL: smulh_v64i16:237; CHECK: // %bb.0:238; CHECK-NEXT: ptrue p0.h, vl64239; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]240; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]241; CHECK-NEXT: smulh z0.h, p0/m, z0.h, z1.h242; CHECK-NEXT: st1h { z0.h }, p0, [x0]243; CHECK-NEXT: ret244 %op1 = load <64 x i16>, ptr %a245 %op2 = load <64 x i16>, ptr %b246 %1 = sext <64 x i16> %op1 to <64 x i32>247 %2 = sext <64 x i16> %op2 to <64 x i32>248 %mul = mul <64 x i32> %1, %2249 %shr = lshr <64 x i32> %mul, splat (i32 16)250 %res = trunc <64 x i32> %shr to <64 x i16>251 store <64 x i16> %res, ptr %a252 ret void253}254 255define void @smulh_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {256; CHECK-LABEL: smulh_v128i16:257; CHECK: // %bb.0:258; CHECK-NEXT: ptrue p0.h, vl128259; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]260; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]261; CHECK-NEXT: smulh z0.h, p0/m, z0.h, z1.h262; CHECK-NEXT: st1h { z0.h }, p0, [x0]263; CHECK-NEXT: ret264 %op1 = load <128 x i16>, ptr %a265 %op2 = load <128 x i16>, ptr %b266 %1 = sext <128 x i16> %op1 to <128 x i32>267 %2 = sext <128 x i16> %op2 to <128 x i32>268 %mul = mul <128 x i32> %1, %2269 %shr = lshr <128 x i32> %mul, splat (i32 16)270 %res = trunc <128 x i32> %shr to <128 x i16>271 store <128 x i16> %res, ptr %a272 ret void273}274 275; Vector i64 multiplications are not legal for NEON so use SVE when available.276define <2 x i32> @smulh_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(2,0) #0 {277; CHECK-LABEL: smulh_v2i32:278; CHECK: // %bb.0:279; CHECK-NEXT: ptrue p0.s, vl2280; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0281; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1282; CHECK-NEXT: smulh z0.s, p0/m, z0.s, z1.s283; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0284; CHECK-NEXT: ret285 %1 = sext <2 x i32> %op1 to <2 x i64>286 %2 = sext <2 x i32> %op2 to <2 x i64>287 %mul = mul <2 x i64> %1, %2288 %shr = lshr <2 x i64> %mul, splat (i64 32)289 %res = trunc <2 x i64> %shr to <2 x i32>290 ret <2 x i32> %res291}292 293; Don't use SVE for 128-bit vectors.294define <4 x i32> @smulh_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(2,0) #0 {295; CHECK-LABEL: smulh_v4i32:296; CHECK: // %bb.0:297; CHECK-NEXT: ptrue p0.s, vl4298; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0299; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1300; CHECK-NEXT: smulh z0.s, p0/m, z0.s, z1.s301; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0302; CHECK-NEXT: ret303 %1 = sext <4 x i32> %op1 to <4 x i64>304 %2 = sext <4 x i32> %op2 to <4 x i64>305 %mul = mul <4 x i64> %1, %2306 %shr = lshr <4 x i64> %mul, splat (i64 32)307 %res = trunc <4 x i64> %shr to <4 x i32>308 ret <4 x i32> %res309}310 311define void @smulh_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {312; CHECK-LABEL: smulh_v8i32:313; CHECK: // %bb.0:314; CHECK-NEXT: ptrue p0.s, vl8315; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]316; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]317; CHECK-NEXT: smulh z0.s, p0/m, z0.s, z1.s318; CHECK-NEXT: st1w { z0.s }, p0, [x0]319; CHECK-NEXT: ret320 %op1 = load <8 x i32>, ptr %a321 %op2 = load <8 x i32>, ptr %b322 %1 = sext <8 x i32> %op1 to <8 x i64>323 %2 = sext <8 x i32> %op2 to <8 x i64>324 %mul = mul <8 x i64> %1, %2325 %shr = lshr <8 x i64> %mul, splat (i64 32)326 %res = trunc <8 x i64> %shr to <8 x i32>327 store <8 x i32> %res, ptr %a328 ret void329}330 331define void @smulh_v16i32(ptr %a, ptr %b) #0 {332; VBITS_GE_256-LABEL: smulh_v16i32:333; VBITS_GE_256: // %bb.0:334; VBITS_GE_256-NEXT: ptrue p0.s, vl8335; VBITS_GE_256-NEXT: mov x8, #8 // =0x8336; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]337; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]338; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0]339; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1]340; VBITS_GE_256-NEXT: smulh z0.s, p0/m, z0.s, z1.s341; VBITS_GE_256-NEXT: smulh z2.s, p0/m, z2.s, z3.s342; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]343; VBITS_GE_256-NEXT: st1w { z2.s }, p0, [x0]344; VBITS_GE_256-NEXT: ret345;346; VBITS_GE_512-LABEL: smulh_v16i32:347; VBITS_GE_512: // %bb.0:348; VBITS_GE_512-NEXT: ptrue p0.s, vl16349; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]350; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]351; VBITS_GE_512-NEXT: smulh z0.s, p0/m, z0.s, z1.s352; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]353; VBITS_GE_512-NEXT: ret354 %op1 = load <16 x i32>, ptr %a355 %op2 = load <16 x i32>, ptr %b356 %1 = sext <16 x i32> %op1 to <16 x i64>357 %2 = sext <16 x i32> %op2 to <16 x i64>358 %mul = mul <16 x i64> %1, %2359 %shr = lshr <16 x i64> %mul, splat (i64 32)360 %res = trunc <16 x i64> %shr to <16 x i32>361 store <16 x i32> %res, ptr %a362 ret void363}364 365define void @smulh_v32i32(ptr %a, ptr %b) vscale_range(8,0) #0 {366; CHECK-LABEL: smulh_v32i32:367; CHECK: // %bb.0:368; CHECK-NEXT: ptrue p0.s, vl32369; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]370; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]371; CHECK-NEXT: smulh z0.s, p0/m, z0.s, z1.s372; CHECK-NEXT: st1w { z0.s }, p0, [x0]373; CHECK-NEXT: ret374 %op1 = load <32 x i32>, ptr %a375 %op2 = load <32 x i32>, ptr %b376 %1 = sext <32 x i32> %op1 to <32 x i64>377 %2 = sext <32 x i32> %op2 to <32 x i64>378 %mul = mul <32 x i64> %1, %2379 %shr = lshr <32 x i64> %mul, splat (i64 32)380 %res = trunc <32 x i64> %shr to <32 x i32>381 store <32 x i32> %res, ptr %a382 ret void383}384 385define void @smulh_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {386; CHECK-LABEL: smulh_v64i32:387; CHECK: // %bb.0:388; CHECK-NEXT: ptrue p0.s, vl64389; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]390; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]391; CHECK-NEXT: smulh z0.s, p0/m, z0.s, z1.s392; CHECK-NEXT: st1w { z0.s }, p0, [x0]393; CHECK-NEXT: ret394 %op1 = load <64 x i32>, ptr %a395 %op2 = load <64 x i32>, ptr %b396 %1 = sext <64 x i32> %op1 to <64 x i64>397 %2 = sext <64 x i32> %op2 to <64 x i64>398 %mul = mul <64 x i64> %1, %2399 %shr = lshr <64 x i64> %mul, splat (i64 32)400 %res = trunc <64 x i64> %shr to <64 x i32>401 store <64 x i32> %res, ptr %a402 ret void403}404 405; Vector i64 multiplications are not legal for NEON so use SVE when available.406define <1 x i64> @smulh_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(2,0) #0 {407; CHECK-LABEL: smulh_v1i64:408; CHECK: // %bb.0:409; CHECK-NEXT: ptrue p0.d, vl1410; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0411; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1412; CHECK-NEXT: smulh z0.d, p0/m, z0.d, z1.d413; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0414; CHECK-NEXT: ret415 %1 = sext <1 x i64> %op1 to <1 x i128>416 %2 = sext <1 x i64> %op2 to <1 x i128>417 %mul = mul <1 x i128> %1, %2418 %shr = lshr <1 x i128> %mul, splat (i128 64)419 %res = trunc <1 x i128> %shr to <1 x i64>420 ret <1 x i64> %res421}422 423; Vector i64 multiplications are not legal for NEON so use SVE when available.424define <2 x i64> @smulh_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(2,0) #0 {425; CHECK-LABEL: smulh_v2i64:426; CHECK: // %bb.0:427; CHECK-NEXT: ptrue p0.d, vl2428; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0429; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1430; CHECK-NEXT: smulh z0.d, p0/m, z0.d, z1.d431; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0432; CHECK-NEXT: ret433 %1 = sext <2 x i64> %op1 to <2 x i128>434 %2 = sext <2 x i64> %op2 to <2 x i128>435 %mul = mul <2 x i128> %1, %2436 %shr = lshr <2 x i128> %mul, splat (i128 64)437 %res = trunc <2 x i128> %shr to <2 x i64>438 ret <2 x i64> %res439}440 441define void @smulh_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {442; CHECK-LABEL: smulh_v4i64:443; CHECK: // %bb.0:444; CHECK-NEXT: ptrue p0.d, vl4445; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]446; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]447; CHECK-NEXT: smulh z0.d, p0/m, z0.d, z1.d448; CHECK-NEXT: st1d { z0.d }, p0, [x0]449; CHECK-NEXT: ret450 %op1 = load <4 x i64>, ptr %a451 %op2 = load <4 x i64>, ptr %b452 %1 = sext <4 x i64> %op1 to <4 x i128>453 %2 = sext <4 x i64> %op2 to <4 x i128>454 %mul = mul <4 x i128> %1, %2455 %shr = lshr <4 x i128> %mul, splat (i128 64)456 %res = trunc <4 x i128> %shr to <4 x i64>457 store <4 x i64> %res, ptr %a458 ret void459}460 461define void @smulh_v8i64(ptr %a, ptr %b) #0 {462; VBITS_GE_256-LABEL: smulh_v8i64:463; VBITS_GE_256: // %bb.0:464; VBITS_GE_256-NEXT: ptrue p0.d, vl4465; VBITS_GE_256-NEXT: mov x8, #4 // =0x4466; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]467; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]468; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0]469; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1]470; VBITS_GE_256-NEXT: smulh z0.d, p0/m, z0.d, z1.d471; VBITS_GE_256-NEXT: smulh z2.d, p0/m, z2.d, z3.d472; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]473; VBITS_GE_256-NEXT: st1d { z2.d }, p0, [x0]474; VBITS_GE_256-NEXT: ret475;476; VBITS_GE_512-LABEL: smulh_v8i64:477; VBITS_GE_512: // %bb.0:478; VBITS_GE_512-NEXT: ptrue p0.d, vl8479; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]480; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]481; VBITS_GE_512-NEXT: smulh z0.d, p0/m, z0.d, z1.d482; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]483; VBITS_GE_512-NEXT: ret484 %op1 = load <8 x i64>, ptr %a485 %op2 = load <8 x i64>, ptr %b486 %1 = sext <8 x i64> %op1 to <8 x i128>487 %2 = sext <8 x i64> %op2 to <8 x i128>488 %mul = mul <8 x i128> %1, %2489 %shr = lshr <8 x i128> %mul, splat (i128 64)490 %res = trunc <8 x i128> %shr to <8 x i64>491 store <8 x i64> %res, ptr %a492 ret void493}494 495define void @smulh_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 {496; CHECK-LABEL: smulh_v16i64:497; CHECK: // %bb.0:498; CHECK-NEXT: ptrue p0.d, vl16499; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]500; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]501; CHECK-NEXT: smulh z0.d, p0/m, z0.d, z1.d502; CHECK-NEXT: st1d { z0.d }, p0, [x0]503; CHECK-NEXT: ret504 %op1 = load <16 x i64>, ptr %a505 %op2 = load <16 x i64>, ptr %b506 %1 = sext <16 x i64> %op1 to <16 x i128>507 %2 = sext <16 x i64> %op2 to <16 x i128>508 %mul = mul <16 x i128> %1, %2509 %shr = lshr <16 x i128> %mul, splat (i128 64)510 %res = trunc <16 x i128> %shr to <16 x i64>511 store <16 x i64> %res, ptr %a512 ret void513}514 515define void @smulh_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {516; CHECK-LABEL: smulh_v32i64:517; CHECK: // %bb.0:518; CHECK-NEXT: ptrue p0.d, vl32519; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]520; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]521; CHECK-NEXT: smulh z0.d, p0/m, z0.d, z1.d522; CHECK-NEXT: st1d { z0.d }, p0, [x0]523; CHECK-NEXT: ret524 %op1 = load <32 x i64>, ptr %a525 %op2 = load <32 x i64>, ptr %b526 %1 = sext <32 x i64> %op1 to <32 x i128>527 %2 = sext <32 x i64> %op2 to <32 x i128>528 %mul = mul <32 x i128> %1, %2529 %shr = lshr <32 x i128> %mul, splat (i128 64)530 %res = trunc <32 x i128> %shr to <32 x i64>531 store <32 x i64> %res, ptr %a532 ret void533}534 535;536; UMULH537;538 539; Don't use SVE for 64-bit vectors.540; FIXME: The codegen for the >=256 bits case can be improved.541define <8 x i8> @umulh_v8i8(<8 x i8> %op1, <8 x i8> %op2) vscale_range(2,0) #0 {542; CHECK-LABEL: umulh_v8i8:543; CHECK: // %bb.0:544; CHECK-NEXT: ptrue p0.b, vl8545; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0546; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1547; CHECK-NEXT: umulh z0.b, p0/m, z0.b, z1.b548; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0549; CHECK-NEXT: ret550 %1 = zext <8 x i8> %op1 to <8 x i16>551 %2 = zext <8 x i8> %op2 to <8 x i16>552 %mul = mul <8 x i16> %1, %2553 %shr = lshr <8 x i16> %mul, splat (i16 8)554 %res = trunc <8 x i16> %shr to <8 x i8>555 ret <8 x i8> %res556}557 558; Don't use SVE for 128-bit vectors.559define <16 x i8> @umulh_v16i8(<16 x i8> %op1, <16 x i8> %op2) vscale_range(2,0) #0 {560; CHECK-LABEL: umulh_v16i8:561; CHECK: // %bb.0:562; CHECK-NEXT: ptrue p0.b, vl16563; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0564; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1565; CHECK-NEXT: umulh z0.b, p0/m, z0.b, z1.b566; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0567; CHECK-NEXT: ret568 %1 = zext <16 x i8> %op1 to <16 x i16>569 %2 = zext <16 x i8> %op2 to <16 x i16>570 %mul = mul <16 x i16> %1, %2571 %shr = lshr <16 x i16> %mul, splat (i16 8)572 %res = trunc <16 x i16> %shr to <16 x i8>573 ret <16 x i8> %res574}575 576define void @umulh_v32i8(ptr %a, ptr %b) vscale_range(2,0) #0 {577; CHECK-LABEL: umulh_v32i8:578; CHECK: // %bb.0:579; CHECK-NEXT: ptrue p0.b, vl32580; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]581; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]582; CHECK-NEXT: umulh z0.b, p0/m, z0.b, z1.b583; CHECK-NEXT: st1b { z0.b }, p0, [x0]584; CHECK-NEXT: ret585 %op1 = load <32 x i8>, ptr %a586 %op2 = load <32 x i8>, ptr %b587 %1 = zext <32 x i8> %op1 to <32 x i16>588 %2 = zext <32 x i8> %op2 to <32 x i16>589 %mul = mul <32 x i16> %1, %2590 %shr = lshr <32 x i16> %mul, splat (i16 8)591 %res = trunc <32 x i16> %shr to <32 x i8>592 store <32 x i8> %res, ptr %a593 ret void594}595 596define void @umulh_v64i8(ptr %a, ptr %b) #0 {597; VBITS_GE_256-LABEL: umulh_v64i8:598; VBITS_GE_256: // %bb.0:599; VBITS_GE_256-NEXT: ptrue p0.b, vl32600; VBITS_GE_256-NEXT: mov w8, #32 // =0x20601; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]602; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x1, x8]603; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0]604; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x1]605; VBITS_GE_256-NEXT: umulh z0.b, p0/m, z0.b, z1.b606; VBITS_GE_256-NEXT: umulh z2.b, p0/m, z2.b, z3.b607; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0, x8]608; VBITS_GE_256-NEXT: st1b { z2.b }, p0, [x0]609; VBITS_GE_256-NEXT: ret610;611; VBITS_GE_512-LABEL: umulh_v64i8:612; VBITS_GE_512: // %bb.0:613; VBITS_GE_512-NEXT: ptrue p0.b, vl64614; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]615; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x1]616; VBITS_GE_512-NEXT: umulh z0.b, p0/m, z0.b, z1.b617; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0]618; VBITS_GE_512-NEXT: ret619 %op1 = load <64 x i8>, ptr %a620 %op2 = load <64 x i8>, ptr %b621 %1 = zext <64 x i8> %op1 to <64 x i16>622 %2 = zext <64 x i8> %op2 to <64 x i16>623 %mul = mul <64 x i16> %1, %2624 %shr = lshr <64 x i16> %mul, splat (i16 8)625 %res = trunc <64 x i16> %shr to <64 x i8>626 store <64 x i8> %res, ptr %a627 ret void628}629 630define void @umulh_v128i8(ptr %a, ptr %b) vscale_range(8,0) #0 {631; CHECK-LABEL: umulh_v128i8:632; CHECK: // %bb.0:633; CHECK-NEXT: ptrue p0.b, vl128634; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]635; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]636; CHECK-NEXT: umulh z0.b, p0/m, z0.b, z1.b637; CHECK-NEXT: st1b { z0.b }, p0, [x0]638; CHECK-NEXT: ret639 %op1 = load <128 x i8>, ptr %a640 %op2 = load <128 x i8>, ptr %b641 %1 = zext <128 x i8> %op1 to <128 x i16>642 %2 = zext <128 x i8> %op2 to <128 x i16>643 %mul = mul <128 x i16> %1, %2644 %shr = lshr <128 x i16> %mul, splat (i16 8)645 %res = trunc <128 x i16> %shr to <128 x i8>646 store <128 x i8> %res, ptr %a647 ret void648}649 650define void @umulh_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {651; CHECK-LABEL: umulh_v256i8:652; CHECK: // %bb.0:653; CHECK-NEXT: ptrue p0.b, vl256654; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]655; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]656; CHECK-NEXT: umulh z0.b, p0/m, z0.b, z1.b657; CHECK-NEXT: st1b { z0.b }, p0, [x0]658; CHECK-NEXT: ret659 %op1 = load <256 x i8>, ptr %a660 %op2 = load <256 x i8>, ptr %b661 %1 = zext <256 x i8> %op1 to <256 x i16>662 %2 = zext <256 x i8> %op2 to <256 x i16>663 %mul = mul <256 x i16> %1, %2664 %shr = lshr <256 x i16> %mul, splat (i16 8)665 %res = trunc <256 x i16> %shr to <256 x i8>666 store <256 x i8> %res, ptr %a667 ret void668}669 670; Don't use SVE for 64-bit vectors.671; FIXME: The codegen for the >=256 bits case can be improved.672define <4 x i16> @umulh_v4i16(<4 x i16> %op1, <4 x i16> %op2) vscale_range(2,0) #0 {673; CHECK-LABEL: umulh_v4i16:674; CHECK: // %bb.0:675; CHECK-NEXT: ptrue p0.h, vl4676; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0677; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1678; CHECK-NEXT: umulh z0.h, p0/m, z0.h, z1.h679; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0680; CHECK-NEXT: ret681 %1 = zext <4 x i16> %op1 to <4 x i32>682 %2 = zext <4 x i16> %op2 to <4 x i32>683 %mul = mul <4 x i32> %1, %2684 %shr = lshr <4 x i32> %mul, splat (i32 16)685 %res = trunc <4 x i32> %shr to <4 x i16>686 ret <4 x i16> %res687}688 689; Don't use SVE for 128-bit vectors.690define <8 x i16> @umulh_v8i16(<8 x i16> %op1, <8 x i16> %op2) vscale_range(2,0) #0 {691; CHECK-LABEL: umulh_v8i16:692; CHECK: // %bb.0:693; CHECK-NEXT: ptrue p0.h, vl8694; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0695; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1696; CHECK-NEXT: umulh z0.h, p0/m, z0.h, z1.h697; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0698; CHECK-NEXT: ret699 %1 = zext <8 x i16> %op1 to <8 x i32>700 %2 = zext <8 x i16> %op2 to <8 x i32>701 %mul = mul <8 x i32> %1, %2702 %shr = lshr <8 x i32> %mul, splat (i32 16)703 %res = trunc <8 x i32> %shr to <8 x i16>704 ret <8 x i16> %res705}706 707define void @umulh_v16i16(ptr %a, ptr %b) vscale_range(2,0) #0 {708; CHECK-LABEL: umulh_v16i16:709; CHECK: // %bb.0:710; CHECK-NEXT: ptrue p0.h, vl16711; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]712; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]713; CHECK-NEXT: umulh z0.h, p0/m, z0.h, z1.h714; CHECK-NEXT: st1h { z0.h }, p0, [x0]715; CHECK-NEXT: ret716 %op1 = load <16 x i16>, ptr %a717 %op2 = load <16 x i16>, ptr %b718 %1 = zext <16 x i16> %op1 to <16 x i32>719 %2 = zext <16 x i16> %op2 to <16 x i32>720 %mul = mul <16 x i32> %1, %2721 %shr = lshr <16 x i32> %mul, splat (i32 16)722 %res = trunc <16 x i32> %shr to <16 x i16>723 store <16 x i16> %res, ptr %a724 ret void725}726 727define void @umulh_v32i16(ptr %a, ptr %b) #0 {728; VBITS_GE_256-LABEL: umulh_v32i16:729; VBITS_GE_256: // %bb.0:730; VBITS_GE_256-NEXT: ptrue p0.h, vl16731; VBITS_GE_256-NEXT: mov x8, #16 // =0x10732; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]733; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1, x8, lsl #1]734; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0]735; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1]736; VBITS_GE_256-NEXT: umulh z0.h, p0/m, z0.h, z1.h737; VBITS_GE_256-NEXT: umulh z2.h, p0/m, z2.h, z3.h738; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]739; VBITS_GE_256-NEXT: st1h { z2.h }, p0, [x0]740; VBITS_GE_256-NEXT: ret741;742; VBITS_GE_512-LABEL: umulh_v32i16:743; VBITS_GE_512: // %bb.0:744; VBITS_GE_512-NEXT: ptrue p0.h, vl32745; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]746; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]747; VBITS_GE_512-NEXT: umulh z0.h, p0/m, z0.h, z1.h748; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]749; VBITS_GE_512-NEXT: ret750 %op1 = load <32 x i16>, ptr %a751 %op2 = load <32 x i16>, ptr %b752 %1 = zext <32 x i16> %op1 to <32 x i32>753 %2 = zext <32 x i16> %op2 to <32 x i32>754 %mul = mul <32 x i32> %1, %2755 %shr = lshr <32 x i32> %mul, splat (i32 16)756 %res = trunc <32 x i32> %shr to <32 x i16>757 store <32 x i16> %res, ptr %a758 ret void759}760 761define void @umulh_v64i16(ptr %a, ptr %b) vscale_range(8,0) #0 {762; CHECK-LABEL: umulh_v64i16:763; CHECK: // %bb.0:764; CHECK-NEXT: ptrue p0.h, vl64765; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]766; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]767; CHECK-NEXT: umulh z0.h, p0/m, z0.h, z1.h768; CHECK-NEXT: st1h { z0.h }, p0, [x0]769; CHECK-NEXT: ret770 %op1 = load <64 x i16>, ptr %a771 %op2 = load <64 x i16>, ptr %b772 %1 = zext <64 x i16> %op1 to <64 x i32>773 %2 = zext <64 x i16> %op2 to <64 x i32>774 %mul = mul <64 x i32> %1, %2775 %shr = lshr <64 x i32> %mul, splat (i32 16)776 %res = trunc <64 x i32> %shr to <64 x i16>777 store <64 x i16> %res, ptr %a778 ret void779}780 781define void @umulh_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {782; CHECK-LABEL: umulh_v128i16:783; CHECK: // %bb.0:784; CHECK-NEXT: ptrue p0.h, vl128785; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]786; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]787; CHECK-NEXT: umulh z0.h, p0/m, z0.h, z1.h788; CHECK-NEXT: st1h { z0.h }, p0, [x0]789; CHECK-NEXT: ret790 %op1 = load <128 x i16>, ptr %a791 %op2 = load <128 x i16>, ptr %b792 %1 = zext <128 x i16> %op1 to <128 x i32>793 %2 = zext <128 x i16> %op2 to <128 x i32>794 %mul = mul <128 x i32> %1, %2795 %shr = lshr <128 x i32> %mul, splat (i32 16)796 %res = trunc <128 x i32> %shr to <128 x i16>797 store <128 x i16> %res, ptr %a798 ret void799}800 801; Vector i64 multiplications are not legal for NEON so use SVE when available.802define <2 x i32> @umulh_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(2,0) #0 {803; CHECK-LABEL: umulh_v2i32:804; CHECK: // %bb.0:805; CHECK-NEXT: ptrue p0.s, vl2806; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0807; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1808; CHECK-NEXT: umulh z0.s, p0/m, z0.s, z1.s809; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0810; CHECK-NEXT: ret811 %1 = zext <2 x i32> %op1 to <2 x i64>812 %2 = zext <2 x i32> %op2 to <2 x i64>813 %mul = mul <2 x i64> %1, %2814 %shr = lshr <2 x i64> %mul, splat (i64 32)815 %res = trunc <2 x i64> %shr to <2 x i32>816 ret <2 x i32> %res817}818 819; Don't use SVE for 128-bit vectors.820define <4 x i32> @umulh_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(2,0) #0 {821; CHECK-LABEL: umulh_v4i32:822; CHECK: // %bb.0:823; CHECK-NEXT: ptrue p0.s, vl4824; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0825; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1826; CHECK-NEXT: umulh z0.s, p0/m, z0.s, z1.s827; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0828; CHECK-NEXT: ret829 %1 = zext <4 x i32> %op1 to <4 x i64>830 %2 = zext <4 x i32> %op2 to <4 x i64>831 %mul = mul <4 x i64> %1, %2832 %shr = lshr <4 x i64> %mul, splat (i64 32)833 %res = trunc <4 x i64> %shr to <4 x i32>834 ret <4 x i32> %res835}836 837define void @umulh_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {838; CHECK-LABEL: umulh_v8i32:839; CHECK: // %bb.0:840; CHECK-NEXT: ptrue p0.s, vl8841; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]842; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]843; CHECK-NEXT: umulh z0.s, p0/m, z0.s, z1.s844; CHECK-NEXT: st1w { z0.s }, p0, [x0]845; CHECK-NEXT: ret846 %op1 = load <8 x i32>, ptr %a847 %op2 = load <8 x i32>, ptr %b848 %1 = zext <8 x i32> %op1 to <8 x i64>849 %2 = zext <8 x i32> %op2 to <8 x i64>850 %mul = mul <8 x i64> %1, %2851 %shr = lshr <8 x i64> %mul, splat (i64 32)852 %res = trunc <8 x i64> %shr to <8 x i32>853 store <8 x i32> %res, ptr %a854 ret void855}856 857define void @umulh_v16i32(ptr %a, ptr %b) #0 {858; VBITS_GE_256-LABEL: umulh_v16i32:859; VBITS_GE_256: // %bb.0:860; VBITS_GE_256-NEXT: ptrue p0.s, vl8861; VBITS_GE_256-NEXT: mov x8, #8 // =0x8862; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]863; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]864; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0]865; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1]866; VBITS_GE_256-NEXT: umulh z0.s, p0/m, z0.s, z1.s867; VBITS_GE_256-NEXT: umulh z2.s, p0/m, z2.s, z3.s868; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]869; VBITS_GE_256-NEXT: st1w { z2.s }, p0, [x0]870; VBITS_GE_256-NEXT: ret871;872; VBITS_GE_512-LABEL: umulh_v16i32:873; VBITS_GE_512: // %bb.0:874; VBITS_GE_512-NEXT: ptrue p0.s, vl16875; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]876; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]877; VBITS_GE_512-NEXT: umulh z0.s, p0/m, z0.s, z1.s878; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]879; VBITS_GE_512-NEXT: ret880 %op1 = load <16 x i32>, ptr %a881 %op2 = load <16 x i32>, ptr %b882 %1 = zext <16 x i32> %op1 to <16 x i64>883 %2 = zext <16 x i32> %op2 to <16 x i64>884 %mul = mul <16 x i64> %1, %2885 %shr = lshr <16 x i64> %mul, splat (i64 32)886 %res = trunc <16 x i64> %shr to <16 x i32>887 store <16 x i32> %res, ptr %a888 ret void889}890 891define void @umulh_v32i32(ptr %a, ptr %b) vscale_range(8,0) #0 {892; CHECK-LABEL: umulh_v32i32:893; CHECK: // %bb.0:894; CHECK-NEXT: ptrue p0.s, vl32895; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]896; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]897; CHECK-NEXT: umulh z0.s, p0/m, z0.s, z1.s898; CHECK-NEXT: st1w { z0.s }, p0, [x0]899; CHECK-NEXT: ret900 %op1 = load <32 x i32>, ptr %a901 %op2 = load <32 x i32>, ptr %b902 %1 = zext <32 x i32> %op1 to <32 x i64>903 %2 = zext <32 x i32> %op2 to <32 x i64>904 %mul = mul <32 x i64> %1, %2905 %shr = lshr <32 x i64> %mul, splat (i64 32)906 %res = trunc <32 x i64> %shr to <32 x i32>907 store <32 x i32> %res, ptr %a908 ret void909}910 911define void @umulh_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {912; CHECK-LABEL: umulh_v64i32:913; CHECK: // %bb.0:914; CHECK-NEXT: ptrue p0.s, vl64915; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]916; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]917; CHECK-NEXT: umulh z0.s, p0/m, z0.s, z1.s918; CHECK-NEXT: st1w { z0.s }, p0, [x0]919; CHECK-NEXT: ret920 %op1 = load <64 x i32>, ptr %a921 %op2 = load <64 x i32>, ptr %b922 %1 = zext <64 x i32> %op1 to <64 x i64>923 %2 = zext <64 x i32> %op2 to <64 x i64>924 %mul = mul <64 x i64> %1, %2925 %shr = lshr <64 x i64> %mul, splat (i64 32)926 %res = trunc <64 x i64> %shr to <64 x i32>927 store <64 x i32> %res, ptr %a928 ret void929}930 931; Vector i64 multiplications are not legal for NEON so use SVE when available.932define <1 x i64> @umulh_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(2,0) #0 {933; CHECK-LABEL: umulh_v1i64:934; CHECK: // %bb.0:935; CHECK-NEXT: ptrue p0.d, vl1936; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0937; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1938; CHECK-NEXT: umulh z0.d, p0/m, z0.d, z1.d939; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0940; CHECK-NEXT: ret941 %1 = zext <1 x i64> %op1 to <1 x i128>942 %2 = zext <1 x i64> %op2 to <1 x i128>943 %mul = mul <1 x i128> %1, %2944 %shr = lshr <1 x i128> %mul, splat (i128 64)945 %res = trunc <1 x i128> %shr to <1 x i64>946 ret <1 x i64> %res947}948 949; Vector i64 multiplications are not legal for NEON so use SVE when available.950define <2 x i64> @umulh_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(2,0) #0 {951; CHECK-LABEL: umulh_v2i64:952; CHECK: // %bb.0:953; CHECK-NEXT: ptrue p0.d, vl2954; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0955; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1956; CHECK-NEXT: umulh z0.d, p0/m, z0.d, z1.d957; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0958; CHECK-NEXT: ret959 %1 = zext <2 x i64> %op1 to <2 x i128>960 %2 = zext <2 x i64> %op2 to <2 x i128>961 %mul = mul <2 x i128> %1, %2962 %shr = lshr <2 x i128> %mul, splat (i128 64)963 %res = trunc <2 x i128> %shr to <2 x i64>964 ret <2 x i64> %res965}966 967define void @umulh_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {968; CHECK-LABEL: umulh_v4i64:969; CHECK: // %bb.0:970; CHECK-NEXT: ptrue p0.d, vl4971; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]972; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]973; CHECK-NEXT: umulh z0.d, p0/m, z0.d, z1.d974; CHECK-NEXT: st1d { z0.d }, p0, [x0]975; CHECK-NEXT: ret976 %op1 = load <4 x i64>, ptr %a977 %op2 = load <4 x i64>, ptr %b978 %1 = zext <4 x i64> %op1 to <4 x i128>979 %2 = zext <4 x i64> %op2 to <4 x i128>980 %mul = mul <4 x i128> %1, %2981 %shr = lshr <4 x i128> %mul, splat (i128 64)982 %res = trunc <4 x i128> %shr to <4 x i64>983 store <4 x i64> %res, ptr %a984 ret void985}986 987define void @umulh_v8i64(ptr %a, ptr %b) #0 {988; VBITS_GE_256-LABEL: umulh_v8i64:989; VBITS_GE_256: // %bb.0:990; VBITS_GE_256-NEXT: ptrue p0.d, vl4991; VBITS_GE_256-NEXT: mov x8, #4 // =0x4992; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]993; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]994; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0]995; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1]996; VBITS_GE_256-NEXT: umulh z0.d, p0/m, z0.d, z1.d997; VBITS_GE_256-NEXT: umulh z2.d, p0/m, z2.d, z3.d998; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]999; VBITS_GE_256-NEXT: st1d { z2.d }, p0, [x0]1000; VBITS_GE_256-NEXT: ret1001;1002; VBITS_GE_512-LABEL: umulh_v8i64:1003; VBITS_GE_512: // %bb.0:1004; VBITS_GE_512-NEXT: ptrue p0.d, vl81005; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]1006; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]1007; VBITS_GE_512-NEXT: umulh z0.d, p0/m, z0.d, z1.d1008; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]1009; VBITS_GE_512-NEXT: ret1010 %op1 = load <8 x i64>, ptr %a1011 %op2 = load <8 x i64>, ptr %b1012 %1 = zext <8 x i64> %op1 to <8 x i128>1013 %2 = zext <8 x i64> %op2 to <8 x i128>1014 %mul = mul <8 x i128> %1, %21015 %shr = lshr <8 x i128> %mul, splat (i128 64)1016 %res = trunc <8 x i128> %shr to <8 x i64>1017 store <8 x i64> %res, ptr %a1018 ret void1019}1020 1021define void @umulh_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 {1022; CHECK-LABEL: umulh_v16i64:1023; CHECK: // %bb.0:1024; CHECK-NEXT: ptrue p0.d, vl161025; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1026; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]1027; CHECK-NEXT: umulh z0.d, p0/m, z0.d, z1.d1028; CHECK-NEXT: st1d { z0.d }, p0, [x0]1029; CHECK-NEXT: ret1030 %op1 = load <16 x i64>, ptr %a1031 %op2 = load <16 x i64>, ptr %b1032 %1 = zext <16 x i64> %op1 to <16 x i128>1033 %2 = zext <16 x i64> %op2 to <16 x i128>1034 %mul = mul <16 x i128> %1, %21035 %shr = lshr <16 x i128> %mul, splat (i128 64)1036 %res = trunc <16 x i128> %shr to <16 x i64>1037 store <16 x i64> %res, ptr %a1038 ret void1039}1040 1041define void @umulh_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {1042; CHECK-LABEL: umulh_v32i64:1043; CHECK: // %bb.0:1044; CHECK-NEXT: ptrue p0.d, vl321045; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1046; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]1047; CHECK-NEXT: umulh z0.d, p0/m, z0.d, z1.d1048; CHECK-NEXT: st1d { z0.d }, p0, [x0]1049; CHECK-NEXT: ret1050 %op1 = load <32 x i64>, ptr %a1051 %op2 = load <32 x i64>, ptr %b1052 %1 = zext <32 x i64> %op1 to <32 x i128>1053 %2 = zext <32 x i64> %op2 to <32 x i128>1054 %mul = mul <32 x i128> %1, %21055 %shr = lshr <32 x i128> %mul, splat (i128 64)1056 %res = trunc <32 x i128> %shr to <32 x i64>1057 store <32 x i64> %res, ptr %a1058 ret void1059}1060attributes #0 = { "target-features"="+sve" }1061