1472 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -aarch64-sve-vector-bits-min=128 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_1283; RUN: llc -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2564; RUN: llc -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5125; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5126 7target triple = "aarch64-unknown-linux-gnu"8 9;10; SDIV11;12 13; Vector vXi8 sdiv are not legal for NEON so use SVE when available.14; FIXME: We should be able to improve the codegen for >= 256 bits here.15define <8 x i8> @sdiv_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 {16; VBITS_GE_128-LABEL: sdiv_v8i8:17; VBITS_GE_128: // %bb.0:18; VBITS_GE_128-NEXT: sshll v1.8h, v1.8b, #019; VBITS_GE_128-NEXT: sshll v0.8h, v0.8b, #020; VBITS_GE_128-NEXT: ptrue p0.s, vl421; VBITS_GE_128-NEXT: sshll2 v2.4s, v1.8h, #022; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #023; VBITS_GE_128-NEXT: sshll v1.4s, v1.4h, #024; VBITS_GE_128-NEXT: sshll v0.4s, v0.4h, #025; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s26; VBITS_GE_128-NEXT: sdiv z0.s, p0/m, z0.s, z1.s27; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v2.8h28; VBITS_GE_128-NEXT: xtn v0.8b, v0.8h29; VBITS_GE_128-NEXT: ret30;31; VBITS_GE_256-LABEL: sdiv_v8i8:32; VBITS_GE_256: // %bb.0:33; VBITS_GE_256-NEXT: // kill: def $d1 killed $d1 def $z134; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 def $z035; VBITS_GE_256-NEXT: ptrue p0.s, vl836; VBITS_GE_256-NEXT: sunpklo z1.h, z1.b37; VBITS_GE_256-NEXT: sunpklo z0.h, z0.b38; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h39; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h40; VBITS_GE_256-NEXT: sdiv z0.s, p0/m, z0.s, z1.s41; VBITS_GE_256-NEXT: uzp1 z1.h, z0.h, z0.h42; VBITS_GE_256-NEXT: umov w8, v1.h[0]43; VBITS_GE_256-NEXT: umov w9, v1.h[1]44; VBITS_GE_256-NEXT: fmov s0, w845; VBITS_GE_256-NEXT: umov w8, v1.h[2]46; VBITS_GE_256-NEXT: mov v0.b[1], w947; VBITS_GE_256-NEXT: mov v0.b[2], w848; VBITS_GE_256-NEXT: umov w8, v1.h[3]49; VBITS_GE_256-NEXT: mov v0.b[3], w850; VBITS_GE_256-NEXT: umov w8, v1.h[4]51; VBITS_GE_256-NEXT: mov v0.b[4], w852; VBITS_GE_256-NEXT: umov w8, v1.h[5]53; VBITS_GE_256-NEXT: mov v0.b[5], w854; VBITS_GE_256-NEXT: umov w8, v1.h[6]55; VBITS_GE_256-NEXT: mov v0.b[6], w856; VBITS_GE_256-NEXT: umov w8, v1.h[7]57; VBITS_GE_256-NEXT: mov v0.b[7], w858; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 killed $q059; VBITS_GE_256-NEXT: ret60;61; VBITS_GE_512-LABEL: sdiv_v8i8:62; VBITS_GE_512: // %bb.0:63; VBITS_GE_512-NEXT: // kill: def $d1 killed $d1 def $z164; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 def $z065; VBITS_GE_512-NEXT: ptrue p0.s, vl866; VBITS_GE_512-NEXT: sunpklo z1.h, z1.b67; VBITS_GE_512-NEXT: sunpklo z0.h, z0.b68; VBITS_GE_512-NEXT: sunpklo z1.s, z1.h69; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h70; VBITS_GE_512-NEXT: sdiv z0.s, p0/m, z0.s, z1.s71; VBITS_GE_512-NEXT: uzp1 z1.h, z0.h, z0.h72; VBITS_GE_512-NEXT: umov w8, v1.h[0]73; VBITS_GE_512-NEXT: umov w9, v1.h[1]74; VBITS_GE_512-NEXT: fmov s0, w875; VBITS_GE_512-NEXT: umov w8, v1.h[2]76; VBITS_GE_512-NEXT: mov v0.b[1], w977; VBITS_GE_512-NEXT: mov v0.b[2], w878; VBITS_GE_512-NEXT: umov w8, v1.h[3]79; VBITS_GE_512-NEXT: mov v0.b[3], w880; VBITS_GE_512-NEXT: umov w8, v1.h[4]81; VBITS_GE_512-NEXT: mov v0.b[4], w882; VBITS_GE_512-NEXT: umov w8, v1.h[5]83; VBITS_GE_512-NEXT: mov v0.b[5], w884; VBITS_GE_512-NEXT: umov w8, v1.h[6]85; VBITS_GE_512-NEXT: mov v0.b[6], w886; VBITS_GE_512-NEXT: umov w8, v1.h[7]87; VBITS_GE_512-NEXT: mov v0.b[7], w888; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 killed $q089; VBITS_GE_512-NEXT: ret90 %res = sdiv <8 x i8> %op1, %op291 ret <8 x i8> %res92}93 94define <16 x i8> @sdiv_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 {95; VBITS_GE_128-LABEL: sdiv_v16i8:96; VBITS_GE_128: // %bb.0:97; VBITS_GE_128-NEXT: sshll2 v2.8h, v1.16b, #098; VBITS_GE_128-NEXT: sshll2 v3.8h, v0.16b, #099; VBITS_GE_128-NEXT: sshll v1.8h, v1.8b, #0100; VBITS_GE_128-NEXT: sshll v0.8h, v0.8b, #0101; VBITS_GE_128-NEXT: ptrue p0.s, vl4102; VBITS_GE_128-NEXT: sshll2 v4.4s, v2.8h, #0103; VBITS_GE_128-NEXT: sshll2 v5.4s, v3.8h, #0104; VBITS_GE_128-NEXT: sshll v2.4s, v2.4h, #0105; VBITS_GE_128-NEXT: sshll v3.4s, v3.4h, #0106; VBITS_GE_128-NEXT: sdivr z4.s, p0/m, z4.s, z5.s107; VBITS_GE_128-NEXT: sshll2 v5.4s, v0.8h, #0108; VBITS_GE_128-NEXT: sshll v0.4s, v0.4h, #0109; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s110; VBITS_GE_128-NEXT: sshll2 v3.4s, v1.8h, #0111; VBITS_GE_128-NEXT: sshll v1.4s, v1.4h, #0112; VBITS_GE_128-NEXT: sdivr z3.s, p0/m, z3.s, z5.s113; VBITS_GE_128-NEXT: sdiv z0.s, p0/m, z0.s, z1.s114; VBITS_GE_128-NEXT: uzp1 v1.8h, v2.8h, v4.8h115; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v3.8h116; VBITS_GE_128-NEXT: uzp1 v0.16b, v0.16b, v1.16b117; VBITS_GE_128-NEXT: ret118;119; VBITS_GE_256-LABEL: sdiv_v16i8:120; VBITS_GE_256: // %bb.0:121; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1122; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0123; VBITS_GE_256-NEXT: ptrue p0.s, vl8124; VBITS_GE_256-NEXT: sunpklo z1.h, z1.b125; VBITS_GE_256-NEXT: sunpklo z0.h, z0.b126; VBITS_GE_256-NEXT: sunpklo z2.s, z1.h127; VBITS_GE_256-NEXT: sunpklo z3.s, z0.h128; VBITS_GE_256-NEXT: ext z1.b, z1.b, z1.b, #16129; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16130; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h131; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h132; VBITS_GE_256-NEXT: sdivr z2.s, p0/m, z2.s, z3.s133; VBITS_GE_256-NEXT: sdiv z0.s, p0/m, z0.s, z1.s134; VBITS_GE_256-NEXT: ptrue p0.h, vl8135; VBITS_GE_256-NEXT: uzp1 z1.h, z2.h, z2.h136; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h137; VBITS_GE_256-NEXT: splice z1.h, p0, z1.h, z0.h138; VBITS_GE_256-NEXT: uzp1 z0.b, z1.b, z1.b139; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0140; VBITS_GE_256-NEXT: ret141;142; VBITS_GE_512-LABEL: sdiv_v16i8:143; VBITS_GE_512: // %bb.0:144; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1145; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0146; VBITS_GE_512-NEXT: ptrue p0.s, vl16147; VBITS_GE_512-NEXT: sunpklo z1.h, z1.b148; VBITS_GE_512-NEXT: sunpklo z0.h, z0.b149; VBITS_GE_512-NEXT: sunpklo z1.s, z1.h150; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h151; VBITS_GE_512-NEXT: sdiv z0.s, p0/m, z0.s, z1.s152; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h153; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b154; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0155; VBITS_GE_512-NEXT: ret156 %res = sdiv <16 x i8> %op1, %op2157 ret <16 x i8> %res158}159 160define void @sdiv_v32i8(ptr %a, ptr %b) vscale_range(8,0) #0 {161; CHECK-LABEL: sdiv_v32i8:162; CHECK: // %bb.0:163; CHECK-NEXT: ptrue p0.b, vl32164; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]165; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]166; CHECK-NEXT: ptrue p0.s, vl32167; CHECK-NEXT: sunpklo z1.h, z1.b168; CHECK-NEXT: sunpklo z0.h, z0.b169; CHECK-NEXT: sunpklo z1.s, z1.h170; CHECK-NEXT: sunpklo z0.s, z0.h171; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s172; CHECK-NEXT: st1b { z0.s }, p0, [x0]173; CHECK-NEXT: ret174 %op1 = load <32 x i8>, ptr %a175 %op2 = load <32 x i8>, ptr %b176 %res = sdiv <32 x i8> %op1, %op2177 store <32 x i8> %res, ptr %a178 ret void179}180 181define void @sdiv_v64i8(ptr %a, ptr %b) vscale_range(16,0) #0 {182; CHECK-LABEL: sdiv_v64i8:183; CHECK: // %bb.0:184; CHECK-NEXT: ptrue p0.b, vl64185; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]186; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]187; CHECK-NEXT: ptrue p0.s, vl64188; CHECK-NEXT: sunpklo z1.h, z1.b189; CHECK-NEXT: sunpklo z0.h, z0.b190; CHECK-NEXT: sunpklo z1.s, z1.h191; CHECK-NEXT: sunpklo z0.s, z0.h192; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s193; CHECK-NEXT: st1b { z0.s }, p0, [x0]194; CHECK-NEXT: ret195 %op1 = load <64 x i8>, ptr %a196 %op2 = load <64 x i8>, ptr %b197 %res = sdiv <64 x i8> %op1, %op2198 store <64 x i8> %res, ptr %a199 ret void200}201 202define void @sdiv_v128i8(ptr %a, ptr %b) vscale_range(16,0) #0 {203; CHECK-LABEL: sdiv_v128i8:204; CHECK: // %bb.0:205; CHECK-NEXT: ptrue p0.b, vl128206; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]207; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]208; CHECK-NEXT: ptrue p0.s, vl64209; CHECK-NEXT: sunpklo z1.h, z1.b210; CHECK-NEXT: sunpklo z0.h, z0.b211; CHECK-NEXT: sunpklo z2.s, z1.h212; CHECK-NEXT: sunpklo z3.s, z0.h213; CHECK-NEXT: ext z1.b, z1.b, z1.b, #128214; CHECK-NEXT: ext z0.b, z0.b, z0.b, #128215; CHECK-NEXT: sunpklo z1.s, z1.h216; CHECK-NEXT: sunpklo z0.s, z0.h217; CHECK-NEXT: sdivr z2.s, p0/m, z2.s, z3.s218; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s219; CHECK-NEXT: ptrue p0.h, vl64220; CHECK-NEXT: uzp1 z1.h, z2.h, z2.h221; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h222; CHECK-NEXT: splice z1.h, p0, z1.h, z0.h223; CHECK-NEXT: ptrue p0.h, vl128224; CHECK-NEXT: st1b { z1.h }, p0, [x0]225; CHECK-NEXT: ret226 %op1 = load <128 x i8>, ptr %a227 %op2 = load <128 x i8>, ptr %b228 %res = sdiv <128 x i8> %op1, %op2229 store <128 x i8> %res, ptr %a230 ret void231}232 233define void @sdiv_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {234; CHECK-LABEL: sdiv_v256i8:235; CHECK: // %bb.0:236; CHECK-NEXT: ptrue p0.b, vl256237; CHECK-NEXT: ptrue p1.s, vl64238; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]239; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]240; CHECK-NEXT: sunpklo z2.h, z1.b241; CHECK-NEXT: sunpklo z3.h, z0.b242; CHECK-NEXT: ext z1.b, z1.b, z1.b, #128243; CHECK-NEXT: ext z0.b, z0.b, z0.b, #128244; CHECK-NEXT: sunpklo z1.h, z1.b245; CHECK-NEXT: sunpklo z4.s, z2.h246; CHECK-NEXT: sunpklo z5.s, z3.h247; CHECK-NEXT: ext z2.b, z2.b, z2.b, #128248; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128249; CHECK-NEXT: sunpklo z0.h, z0.b250; CHECK-NEXT: sunpklo z2.s, z2.h251; CHECK-NEXT: sunpklo z3.s, z3.h252; CHECK-NEXT: sdivr z4.s, p1/m, z4.s, z5.s253; CHECK-NEXT: sunpklo z5.s, z0.h254; CHECK-NEXT: ext z0.b, z0.b, z0.b, #128255; CHECK-NEXT: sunpklo z0.s, z0.h256; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s257; CHECK-NEXT: sunpklo z3.s, z1.h258; CHECK-NEXT: ext z1.b, z1.b, z1.b, #128259; CHECK-NEXT: sunpklo z1.s, z1.h260; CHECK-NEXT: sdivr z3.s, p1/m, z3.s, z5.s261; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h262; CHECK-NEXT: sdiv z0.s, p1/m, z0.s, z1.s263; CHECK-NEXT: uzp1 z1.h, z4.h, z4.h264; CHECK-NEXT: ptrue p1.h, vl64265; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h266; CHECK-NEXT: splice z1.h, p1, z1.h, z2.h267; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h268; CHECK-NEXT: splice z3.h, p1, z3.h, z0.h269; CHECK-NEXT: uzp1 z0.b, z1.b, z1.b270; CHECK-NEXT: ptrue p1.b, vl128271; CHECK-NEXT: uzp1 z1.b, z3.b, z3.b272; CHECK-NEXT: splice z0.b, p1, z0.b, z1.b273; CHECK-NEXT: st1b { z0.b }, p0, [x0]274; CHECK-NEXT: ret275 %op1 = load <256 x i8>, ptr %a276 %op2 = load <256 x i8>, ptr %b277 %res = sdiv <256 x i8> %op1, %op2278 store <256 x i8> %res, ptr %a279 ret void280}281 282; Vector vXi16 sdiv are not legal for NEON so use SVE when available.283; FIXME: We should be able to improve the codegen for >= 256 bits here.284define <4 x i16> @sdiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 {285; VBITS_GE_128-LABEL: sdiv_v4i16:286; VBITS_GE_128: // %bb.0:287; VBITS_GE_128-NEXT: sshll v1.4s, v1.4h, #0288; VBITS_GE_128-NEXT: sshll v0.4s, v0.4h, #0289; VBITS_GE_128-NEXT: ptrue p0.s, vl4290; VBITS_GE_128-NEXT: sdiv z0.s, p0/m, z0.s, z1.s291; VBITS_GE_128-NEXT: xtn v0.4h, v0.4s292; VBITS_GE_128-NEXT: ret293;294; VBITS_GE_256-LABEL: sdiv_v4i16:295; VBITS_GE_256: // %bb.0:296; VBITS_GE_256-NEXT: sshll v1.4s, v1.4h, #0297; VBITS_GE_256-NEXT: sshll v0.4s, v0.4h, #0298; VBITS_GE_256-NEXT: ptrue p0.s, vl4299; VBITS_GE_256-NEXT: sdivr z1.s, p0/m, z1.s, z0.s300; VBITS_GE_256-NEXT: mov w8, v1.s[1]301; VBITS_GE_256-NEXT: mov v0.16b, v1.16b302; VBITS_GE_256-NEXT: mov w9, v1.s[2]303; VBITS_GE_256-NEXT: mov v0.h[1], w8304; VBITS_GE_256-NEXT: mov w8, v1.s[3]305; VBITS_GE_256-NEXT: mov v0.h[2], w9306; VBITS_GE_256-NEXT: mov v0.h[3], w8307; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 killed $q0308; VBITS_GE_256-NEXT: ret309;310; VBITS_GE_512-LABEL: sdiv_v4i16:311; VBITS_GE_512: // %bb.0:312; VBITS_GE_512-NEXT: sshll v1.4s, v1.4h, #0313; VBITS_GE_512-NEXT: sshll v0.4s, v0.4h, #0314; VBITS_GE_512-NEXT: ptrue p0.s, vl4315; VBITS_GE_512-NEXT: sdivr z1.s, p0/m, z1.s, z0.s316; VBITS_GE_512-NEXT: mov w8, v1.s[1]317; VBITS_GE_512-NEXT: mov v0.16b, v1.16b318; VBITS_GE_512-NEXT: mov w9, v1.s[2]319; VBITS_GE_512-NEXT: mov v0.h[1], w8320; VBITS_GE_512-NEXT: mov w8, v1.s[3]321; VBITS_GE_512-NEXT: mov v0.h[2], w9322; VBITS_GE_512-NEXT: mov v0.h[3], w8323; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 killed $q0324; VBITS_GE_512-NEXT: ret325 %res = sdiv <4 x i16> %op1, %op2326 ret <4 x i16> %res327}328 329define <8 x i16> @sdiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 {330; VBITS_GE_128-LABEL: sdiv_v8i16:331; VBITS_GE_128: // %bb.0:332; VBITS_GE_128-NEXT: sshll2 v2.4s, v1.8h, #0333; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0334; VBITS_GE_128-NEXT: sshll v1.4s, v1.4h, #0335; VBITS_GE_128-NEXT: sshll v0.4s, v0.4h, #0336; VBITS_GE_128-NEXT: ptrue p0.s, vl4337; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s338; VBITS_GE_128-NEXT: sdiv z0.s, p0/m, z0.s, z1.s339; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v2.8h340; VBITS_GE_128-NEXT: ret341;342; VBITS_GE_256-LABEL: sdiv_v8i16:343; VBITS_GE_256: // %bb.0:344; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1345; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0346; VBITS_GE_256-NEXT: ptrue p0.s, vl8347; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h348; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h349; VBITS_GE_256-NEXT: sdiv z0.s, p0/m, z0.s, z1.s350; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h351; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0352; VBITS_GE_256-NEXT: ret353;354; VBITS_GE_512-LABEL: sdiv_v8i16:355; VBITS_GE_512: // %bb.0:356; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1357; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0358; VBITS_GE_512-NEXT: ptrue p0.s, vl8359; VBITS_GE_512-NEXT: sunpklo z1.s, z1.h360; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h361; VBITS_GE_512-NEXT: sdiv z0.s, p0/m, z0.s, z1.s362; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h363; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0364; VBITS_GE_512-NEXT: ret365 %res = sdiv <8 x i16> %op1, %op2366 ret <8 x i16> %res367}368 369define void @sdiv_v16i16(ptr %a, ptr %b) #0 {370; VBITS_GE_128-LABEL: sdiv_v16i16:371; VBITS_GE_128: // %bb.0:372; VBITS_GE_128-NEXT: ldp q4, q1, [x1]373; VBITS_GE_128-NEXT: ptrue p0.s, vl4374; VBITS_GE_128-NEXT: ldr q0, [x0, #16]375; VBITS_GE_128-NEXT: sshll2 v2.4s, v1.8h, #0376; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0377; VBITS_GE_128-NEXT: sshll2 v5.4s, v4.8h, #0378; VBITS_GE_128-NEXT: sshll v4.4s, v4.4h, #0379; VBITS_GE_128-NEXT: sshll v1.4s, v1.4h, #0380; VBITS_GE_128-NEXT: sshll v0.4s, v0.4h, #0381; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s382; VBITS_GE_128-NEXT: ldr q3, [x0]383; VBITS_GE_128-NEXT: sshll2 v6.4s, v3.8h, #0384; VBITS_GE_128-NEXT: sshll v3.4s, v3.4h, #0385; VBITS_GE_128-NEXT: sdivr z5.s, p0/m, z5.s, z6.s386; VBITS_GE_128-NEXT: sdiv z3.s, p0/m, z3.s, z4.s387; VBITS_GE_128-NEXT: sdiv z0.s, p0/m, z0.s, z1.s388; VBITS_GE_128-NEXT: uzp1 v1.8h, v3.8h, v5.8h389; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v2.8h390; VBITS_GE_128-NEXT: stp q1, q0, [x0]391; VBITS_GE_128-NEXT: ret392;393; VBITS_GE_256-LABEL: sdiv_v16i16:394; VBITS_GE_256: // %bb.0:395; VBITS_GE_256-NEXT: ptrue p0.h, vl16396; VBITS_GE_256-NEXT: ptrue p1.s, vl8397; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]398; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1]399; VBITS_GE_256-NEXT: sunpklo z2.s, z1.h400; VBITS_GE_256-NEXT: sunpklo z3.s, z0.h401; VBITS_GE_256-NEXT: ext z1.b, z1.b, z1.b, #16402; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16403; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h404; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h405; VBITS_GE_256-NEXT: sdivr z2.s, p1/m, z2.s, z3.s406; VBITS_GE_256-NEXT: sdiv z0.s, p1/m, z0.s, z1.s407; VBITS_GE_256-NEXT: ptrue p1.h, vl8408; VBITS_GE_256-NEXT: uzp1 z1.h, z2.h, z2.h409; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h410; VBITS_GE_256-NEXT: splice z1.h, p1, z1.h, z0.h411; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]412; VBITS_GE_256-NEXT: ret413;414; VBITS_GE_512-LABEL: sdiv_v16i16:415; VBITS_GE_512: // %bb.0:416; VBITS_GE_512-NEXT: ptrue p0.h, vl16417; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]418; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]419; VBITS_GE_512-NEXT: ptrue p0.s, vl16420; VBITS_GE_512-NEXT: sunpklo z1.s, z1.h421; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h422; VBITS_GE_512-NEXT: sdiv z0.s, p0/m, z0.s, z1.s423; VBITS_GE_512-NEXT: st1h { z0.s }, p0, [x0]424; VBITS_GE_512-NEXT: ret425 %op1 = load <16 x i16>, ptr %a426 %op2 = load <16 x i16>, ptr %b427 %res = sdiv <16 x i16> %op1, %op2428 store <16 x i16> %res, ptr %a429 ret void430}431 432define void @sdiv_v32i16(ptr %a, ptr %b) vscale_range(8,0) #0 {433; CHECK-LABEL: sdiv_v32i16:434; CHECK: // %bb.0:435; CHECK-NEXT: ptrue p0.h, vl32436; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]437; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]438; CHECK-NEXT: ptrue p0.s, vl32439; CHECK-NEXT: sunpklo z1.s, z1.h440; CHECK-NEXT: sunpklo z0.s, z0.h441; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s442; CHECK-NEXT: st1h { z0.s }, p0, [x0]443; CHECK-NEXT: ret444 %op1 = load <32 x i16>, ptr %a445 %op2 = load <32 x i16>, ptr %b446 %res = sdiv <32 x i16> %op1, %op2447 store <32 x i16> %res, ptr %a448 ret void449}450 451define void @sdiv_v64i16(ptr %a, ptr %b) vscale_range(16,0) #0 {452; CHECK-LABEL: sdiv_v64i16:453; CHECK: // %bb.0:454; CHECK-NEXT: ptrue p0.h, vl64455; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]456; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]457; CHECK-NEXT: ptrue p0.s, vl64458; CHECK-NEXT: sunpklo z1.s, z1.h459; CHECK-NEXT: sunpklo z0.s, z0.h460; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s461; CHECK-NEXT: st1h { z0.s }, p0, [x0]462; CHECK-NEXT: ret463 %op1 = load <64 x i16>, ptr %a464 %op2 = load <64 x i16>, ptr %b465 %res = sdiv <64 x i16> %op1, %op2466 store <64 x i16> %res, ptr %a467 ret void468}469 470define void @sdiv_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {471; CHECK-LABEL: sdiv_v128i16:472; CHECK: // %bb.0:473; CHECK-NEXT: ptrue p0.h, vl128474; CHECK-NEXT: ptrue p1.s, vl64475; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]476; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]477; CHECK-NEXT: sunpklo z2.s, z1.h478; CHECK-NEXT: sunpklo z3.s, z0.h479; CHECK-NEXT: ext z1.b, z1.b, z1.b, #128480; CHECK-NEXT: ext z0.b, z0.b, z0.b, #128481; CHECK-NEXT: sunpklo z1.s, z1.h482; CHECK-NEXT: sunpklo z0.s, z0.h483; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s484; CHECK-NEXT: sdiv z0.s, p1/m, z0.s, z1.s485; CHECK-NEXT: ptrue p1.h, vl64486; CHECK-NEXT: uzp1 z1.h, z2.h, z2.h487; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h488; CHECK-NEXT: splice z1.h, p1, z1.h, z0.h489; CHECK-NEXT: st1h { z1.h }, p0, [x0]490; CHECK-NEXT: ret491 %op1 = load <128 x i16>, ptr %a492 %op2 = load <128 x i16>, ptr %b493 %res = sdiv <128 x i16> %op1, %op2494 store <128 x i16> %res, ptr %a495 ret void496}497 498; Vector v2i32 sdiv are not legal for NEON so use SVE when available.499define <2 x i32> @sdiv_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(1,0) #0 {500; CHECK-LABEL: sdiv_v2i32:501; CHECK: // %bb.0:502; CHECK-NEXT: ptrue p0.s, vl2503; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0504; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1505; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s506; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0507; CHECK-NEXT: ret508 %res = sdiv <2 x i32> %op1, %op2509 ret <2 x i32> %res510}511 512; Vector v4i32 sdiv are not legal for NEON so use SVE when available.513define <4 x i32> @sdiv_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(1,0) #0 {514; CHECK-LABEL: sdiv_v4i32:515; CHECK: // %bb.0:516; CHECK-NEXT: ptrue p0.s, vl4517; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0518; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1519; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s520; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0521; CHECK-NEXT: ret522 %res = sdiv <4 x i32> %op1, %op2523 ret <4 x i32> %res524}525 526define void @sdiv_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {527; CHECK-LABEL: sdiv_v8i32:528; CHECK: // %bb.0:529; CHECK-NEXT: ptrue p0.s, vl8530; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]531; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]532; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s533; CHECK-NEXT: st1w { z0.s }, p0, [x0]534; CHECK-NEXT: ret535 %op1 = load <8 x i32>, ptr %a536 %op2 = load <8 x i32>, ptr %b537 %res = sdiv <8 x i32> %op1, %op2538 store <8 x i32> %res, ptr %a539 ret void540}541 542define void @sdiv_v16i32(ptr %a, ptr %b) #0 {543; VBITS_GE_128-LABEL: sdiv_v16i32:544; VBITS_GE_128: // %bb.0:545; VBITS_GE_128-NEXT: ldp q0, q3, [x1]546; VBITS_GE_128-NEXT: ptrue p0.s, vl4547; VBITS_GE_128-NEXT: ldp q1, q2, [x0]548; VBITS_GE_128-NEXT: ldp q5, q4, [x1, #32]549; VBITS_GE_128-NEXT: sdivr z0.s, p0/m, z0.s, z1.s550; VBITS_GE_128-NEXT: ldr q1, [x0, #48]551; VBITS_GE_128-NEXT: sdiv z1.s, p0/m, z1.s, z4.s552; VBITS_GE_128-NEXT: ldr q4, [x0, #32]553; VBITS_GE_128-NEXT: sdiv z4.s, p0/m, z4.s, z5.s554; VBITS_GE_128-NEXT: sdiv z2.s, p0/m, z2.s, z3.s555; VBITS_GE_128-NEXT: stp q4, q1, [x0, #32]556; VBITS_GE_128-NEXT: stp q0, q2, [x0]557; VBITS_GE_128-NEXT: ret558;559; VBITS_GE_256-LABEL: sdiv_v16i32:560; VBITS_GE_256: // %bb.0:561; VBITS_GE_256-NEXT: ptrue p0.s, vl8562; VBITS_GE_256-NEXT: mov x8, #8 // =0x8563; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]564; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]565; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x1]566; VBITS_GE_256-NEXT: sdiv z0.s, p0/m, z0.s, z1.s567; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]568; VBITS_GE_256-NEXT: sdiv z1.s, p0/m, z1.s, z2.s569; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]570; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]571; VBITS_GE_256-NEXT: ret572;573; VBITS_GE_512-LABEL: sdiv_v16i32:574; VBITS_GE_512: // %bb.0:575; VBITS_GE_512-NEXT: ptrue p0.s, vl16576; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]577; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]578; VBITS_GE_512-NEXT: sdiv z0.s, p0/m, z0.s, z1.s579; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]580; VBITS_GE_512-NEXT: ret581 %op1 = load <16 x i32>, ptr %a582 %op2 = load <16 x i32>, ptr %b583 %res = sdiv <16 x i32> %op1, %op2584 store <16 x i32> %res, ptr %a585 ret void586}587 588define void @sdiv_v32i32(ptr %a, ptr %b) vscale_range(8,0) #0 {589; CHECK-LABEL: sdiv_v32i32:590; CHECK: // %bb.0:591; CHECK-NEXT: ptrue p0.s, vl32592; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]593; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]594; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s595; CHECK-NEXT: st1w { z0.s }, p0, [x0]596; CHECK-NEXT: ret597 %op1 = load <32 x i32>, ptr %a598 %op2 = load <32 x i32>, ptr %b599 %res = sdiv <32 x i32> %op1, %op2600 store <32 x i32> %res, ptr %a601 ret void602}603 604define void @sdiv_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {605; CHECK-LABEL: sdiv_v64i32:606; CHECK: // %bb.0:607; CHECK-NEXT: ptrue p0.s, vl64608; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]609; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]610; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s611; CHECK-NEXT: st1w { z0.s }, p0, [x0]612; CHECK-NEXT: ret613 %op1 = load <64 x i32>, ptr %a614 %op2 = load <64 x i32>, ptr %b615 %res = sdiv <64 x i32> %op1, %op2616 store <64 x i32> %res, ptr %a617 ret void618}619 620; Vector i64 sdiv are not legal for NEON so use SVE when available.621define <1 x i64> @sdiv_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(1,0) #0 {622; CHECK-LABEL: sdiv_v1i64:623; CHECK: // %bb.0:624; CHECK-NEXT: ptrue p0.d, vl1625; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0626; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1627; CHECK-NEXT: sdiv z0.d, p0/m, z0.d, z1.d628; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0629; CHECK-NEXT: ret630 %res = sdiv <1 x i64> %op1, %op2631 ret <1 x i64> %res632}633 634; Vector i64 sdiv are not legal for NEON so use SVE when available.635define <2 x i64> @sdiv_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(1,0) #0 {636; CHECK-LABEL: sdiv_v2i64:637; CHECK: // %bb.0:638; CHECK-NEXT: ptrue p0.d, vl2639; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0640; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1641; CHECK-NEXT: sdiv z0.d, p0/m, z0.d, z1.d642; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0643; CHECK-NEXT: ret644 %res = sdiv <2 x i64> %op1, %op2645 ret <2 x i64> %res646}647 648define void @sdiv_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {649; CHECK-LABEL: sdiv_v4i64:650; CHECK: // %bb.0:651; CHECK-NEXT: ptrue p0.d, vl4652; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]653; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]654; CHECK-NEXT: sdiv z0.d, p0/m, z0.d, z1.d655; CHECK-NEXT: st1d { z0.d }, p0, [x0]656; CHECK-NEXT: ret657 %op1 = load <4 x i64>, ptr %a658 %op2 = load <4 x i64>, ptr %b659 %res = sdiv <4 x i64> %op1, %op2660 store <4 x i64> %res, ptr %a661 ret void662}663 664define void @sdiv_v8i64(ptr %a, ptr %b) #0 {665; VBITS_GE_128-LABEL: sdiv_v8i64:666; VBITS_GE_128: // %bb.0:667; VBITS_GE_128-NEXT: ldp q0, q3, [x1]668; VBITS_GE_128-NEXT: ptrue p0.d, vl2669; VBITS_GE_128-NEXT: ldp q1, q2, [x0]670; VBITS_GE_128-NEXT: ldp q5, q4, [x1, #32]671; VBITS_GE_128-NEXT: sdivr z0.d, p0/m, z0.d, z1.d672; VBITS_GE_128-NEXT: ldr q1, [x0, #48]673; VBITS_GE_128-NEXT: sdiv z1.d, p0/m, z1.d, z4.d674; VBITS_GE_128-NEXT: ldr q4, [x0, #32]675; VBITS_GE_128-NEXT: sdiv z4.d, p0/m, z4.d, z5.d676; VBITS_GE_128-NEXT: sdiv z2.d, p0/m, z2.d, z3.d677; VBITS_GE_128-NEXT: stp q4, q1, [x0, #32]678; VBITS_GE_128-NEXT: stp q0, q2, [x0]679; VBITS_GE_128-NEXT: ret680;681; VBITS_GE_256-LABEL: sdiv_v8i64:682; VBITS_GE_256: // %bb.0:683; VBITS_GE_256-NEXT: ptrue p0.d, vl4684; VBITS_GE_256-NEXT: mov x8, #4 // =0x4685; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]686; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]687; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1]688; VBITS_GE_256-NEXT: sdiv z0.d, p0/m, z0.d, z1.d689; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]690; VBITS_GE_256-NEXT: sdiv z1.d, p0/m, z1.d, z2.d691; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]692; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]693; VBITS_GE_256-NEXT: ret694;695; VBITS_GE_512-LABEL: sdiv_v8i64:696; VBITS_GE_512: // %bb.0:697; VBITS_GE_512-NEXT: ptrue p0.d, vl8698; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]699; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]700; VBITS_GE_512-NEXT: sdiv z0.d, p0/m, z0.d, z1.d701; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]702; VBITS_GE_512-NEXT: ret703 %op1 = load <8 x i64>, ptr %a704 %op2 = load <8 x i64>, ptr %b705 %res = sdiv <8 x i64> %op1, %op2706 store <8 x i64> %res, ptr %a707 ret void708}709 710define void @sdiv_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 {711; CHECK-LABEL: sdiv_v16i64:712; CHECK: // %bb.0:713; CHECK-NEXT: ptrue p0.d, vl16714; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]715; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]716; CHECK-NEXT: sdiv z0.d, p0/m, z0.d, z1.d717; CHECK-NEXT: st1d { z0.d }, p0, [x0]718; CHECK-NEXT: ret719 %op1 = load <16 x i64>, ptr %a720 %op2 = load <16 x i64>, ptr %b721 %res = sdiv <16 x i64> %op1, %op2722 store <16 x i64> %res, ptr %a723 ret void724}725 726define void @sdiv_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {727; CHECK-LABEL: sdiv_v32i64:728; CHECK: // %bb.0:729; CHECK-NEXT: ptrue p0.d, vl32730; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]731; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]732; CHECK-NEXT: sdiv z0.d, p0/m, z0.d, z1.d733; CHECK-NEXT: st1d { z0.d }, p0, [x0]734; CHECK-NEXT: ret735 %op1 = load <32 x i64>, ptr %a736 %op2 = load <32 x i64>, ptr %b737 %res = sdiv <32 x i64> %op1, %op2738 store <32 x i64> %res, ptr %a739 ret void740}741 742;743; UDIV744;745 746; Vector vXi8 udiv are not legal for NEON so use SVE when available.747; FIXME: We should be able to improve the codegen for >= 256 bits here.748define <8 x i8> @udiv_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 {749; VBITS_GE_128-LABEL: udiv_v8i8:750; VBITS_GE_128: // %bb.0:751; VBITS_GE_128-NEXT: ushll v1.8h, v1.8b, #0752; VBITS_GE_128-NEXT: ushll v0.8h, v0.8b, #0753; VBITS_GE_128-NEXT: ptrue p0.s, vl4754; VBITS_GE_128-NEXT: ushll2 v2.4s, v1.8h, #0755; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #0756; VBITS_GE_128-NEXT: ushll v1.4s, v1.4h, #0757; VBITS_GE_128-NEXT: ushll v0.4s, v0.4h, #0758; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s759; VBITS_GE_128-NEXT: udiv z0.s, p0/m, z0.s, z1.s760; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v2.8h761; VBITS_GE_128-NEXT: xtn v0.8b, v0.8h762; VBITS_GE_128-NEXT: ret763;764; VBITS_GE_256-LABEL: udiv_v8i8:765; VBITS_GE_256: // %bb.0:766; VBITS_GE_256-NEXT: // kill: def $d1 killed $d1 def $z1767; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 def $z0768; VBITS_GE_256-NEXT: ptrue p0.s, vl8769; VBITS_GE_256-NEXT: uunpklo z1.h, z1.b770; VBITS_GE_256-NEXT: uunpklo z0.h, z0.b771; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h772; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h773; VBITS_GE_256-NEXT: udiv z0.s, p0/m, z0.s, z1.s774; VBITS_GE_256-NEXT: uzp1 z1.h, z0.h, z0.h775; VBITS_GE_256-NEXT: umov w8, v1.h[0]776; VBITS_GE_256-NEXT: umov w9, v1.h[1]777; VBITS_GE_256-NEXT: fmov s0, w8778; VBITS_GE_256-NEXT: umov w8, v1.h[2]779; VBITS_GE_256-NEXT: mov v0.b[1], w9780; VBITS_GE_256-NEXT: mov v0.b[2], w8781; VBITS_GE_256-NEXT: umov w8, v1.h[3]782; VBITS_GE_256-NEXT: mov v0.b[3], w8783; VBITS_GE_256-NEXT: umov w8, v1.h[4]784; VBITS_GE_256-NEXT: mov v0.b[4], w8785; VBITS_GE_256-NEXT: umov w8, v1.h[5]786; VBITS_GE_256-NEXT: mov v0.b[5], w8787; VBITS_GE_256-NEXT: umov w8, v1.h[6]788; VBITS_GE_256-NEXT: mov v0.b[6], w8789; VBITS_GE_256-NEXT: umov w8, v1.h[7]790; VBITS_GE_256-NEXT: mov v0.b[7], w8791; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 killed $q0792; VBITS_GE_256-NEXT: ret793;794; VBITS_GE_512-LABEL: udiv_v8i8:795; VBITS_GE_512: // %bb.0:796; VBITS_GE_512-NEXT: // kill: def $d1 killed $d1 def $z1797; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 def $z0798; VBITS_GE_512-NEXT: ptrue p0.s, vl8799; VBITS_GE_512-NEXT: uunpklo z1.h, z1.b800; VBITS_GE_512-NEXT: uunpklo z0.h, z0.b801; VBITS_GE_512-NEXT: uunpklo z1.s, z1.h802; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h803; VBITS_GE_512-NEXT: udiv z0.s, p0/m, z0.s, z1.s804; VBITS_GE_512-NEXT: uzp1 z1.h, z0.h, z0.h805; VBITS_GE_512-NEXT: umov w8, v1.h[0]806; VBITS_GE_512-NEXT: umov w9, v1.h[1]807; VBITS_GE_512-NEXT: fmov s0, w8808; VBITS_GE_512-NEXT: umov w8, v1.h[2]809; VBITS_GE_512-NEXT: mov v0.b[1], w9810; VBITS_GE_512-NEXT: mov v0.b[2], w8811; VBITS_GE_512-NEXT: umov w8, v1.h[3]812; VBITS_GE_512-NEXT: mov v0.b[3], w8813; VBITS_GE_512-NEXT: umov w8, v1.h[4]814; VBITS_GE_512-NEXT: mov v0.b[4], w8815; VBITS_GE_512-NEXT: umov w8, v1.h[5]816; VBITS_GE_512-NEXT: mov v0.b[5], w8817; VBITS_GE_512-NEXT: umov w8, v1.h[6]818; VBITS_GE_512-NEXT: mov v0.b[6], w8819; VBITS_GE_512-NEXT: umov w8, v1.h[7]820; VBITS_GE_512-NEXT: mov v0.b[7], w8821; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 killed $q0822; VBITS_GE_512-NEXT: ret823 %res = udiv <8 x i8> %op1, %op2824 ret <8 x i8> %res825}826 827define <16 x i8> @udiv_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 {828; VBITS_GE_128-LABEL: udiv_v16i8:829; VBITS_GE_128: // %bb.0:830; VBITS_GE_128-NEXT: ushll2 v2.8h, v1.16b, #0831; VBITS_GE_128-NEXT: ushll2 v3.8h, v0.16b, #0832; VBITS_GE_128-NEXT: ushll v1.8h, v1.8b, #0833; VBITS_GE_128-NEXT: ushll v0.8h, v0.8b, #0834; VBITS_GE_128-NEXT: ptrue p0.s, vl4835; VBITS_GE_128-NEXT: ushll2 v4.4s, v2.8h, #0836; VBITS_GE_128-NEXT: ushll2 v5.4s, v3.8h, #0837; VBITS_GE_128-NEXT: ushll v2.4s, v2.4h, #0838; VBITS_GE_128-NEXT: ushll v3.4s, v3.4h, #0839; VBITS_GE_128-NEXT: udivr z4.s, p0/m, z4.s, z5.s840; VBITS_GE_128-NEXT: ushll2 v5.4s, v0.8h, #0841; VBITS_GE_128-NEXT: ushll v0.4s, v0.4h, #0842; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s843; VBITS_GE_128-NEXT: ushll2 v3.4s, v1.8h, #0844; VBITS_GE_128-NEXT: ushll v1.4s, v1.4h, #0845; VBITS_GE_128-NEXT: udivr z3.s, p0/m, z3.s, z5.s846; VBITS_GE_128-NEXT: udiv z0.s, p0/m, z0.s, z1.s847; VBITS_GE_128-NEXT: uzp1 v1.8h, v2.8h, v4.8h848; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v3.8h849; VBITS_GE_128-NEXT: uzp1 v0.16b, v0.16b, v1.16b850; VBITS_GE_128-NEXT: ret851;852; VBITS_GE_256-LABEL: udiv_v16i8:853; VBITS_GE_256: // %bb.0:854; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1855; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0856; VBITS_GE_256-NEXT: ptrue p0.s, vl8857; VBITS_GE_256-NEXT: uunpklo z1.h, z1.b858; VBITS_GE_256-NEXT: uunpklo z0.h, z0.b859; VBITS_GE_256-NEXT: uunpklo z2.s, z1.h860; VBITS_GE_256-NEXT: uunpklo z3.s, z0.h861; VBITS_GE_256-NEXT: ext z1.b, z1.b, z1.b, #16862; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16863; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h864; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h865; VBITS_GE_256-NEXT: udivr z2.s, p0/m, z2.s, z3.s866; VBITS_GE_256-NEXT: udiv z0.s, p0/m, z0.s, z1.s867; VBITS_GE_256-NEXT: ptrue p0.h, vl8868; VBITS_GE_256-NEXT: uzp1 z1.h, z2.h, z2.h869; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h870; VBITS_GE_256-NEXT: splice z1.h, p0, z1.h, z0.h871; VBITS_GE_256-NEXT: uzp1 z0.b, z1.b, z1.b872; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0873; VBITS_GE_256-NEXT: ret874;875; VBITS_GE_512-LABEL: udiv_v16i8:876; VBITS_GE_512: // %bb.0:877; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1878; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0879; VBITS_GE_512-NEXT: ptrue p0.s, vl16880; VBITS_GE_512-NEXT: uunpklo z1.h, z1.b881; VBITS_GE_512-NEXT: uunpklo z0.h, z0.b882; VBITS_GE_512-NEXT: uunpklo z1.s, z1.h883; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h884; VBITS_GE_512-NEXT: udiv z0.s, p0/m, z0.s, z1.s885; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h886; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b887; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0888; VBITS_GE_512-NEXT: ret889 %res = udiv <16 x i8> %op1, %op2890 ret <16 x i8> %res891}892 893define void @udiv_v32i8(ptr %a, ptr %b) vscale_range(8,0) #0 {894; CHECK-LABEL: udiv_v32i8:895; CHECK: // %bb.0:896; CHECK-NEXT: ptrue p0.s, vl32897; CHECK-NEXT: ld1b { z0.s }, p0/z, [x1]898; CHECK-NEXT: ld1b { z1.s }, p0/z, [x0]899; CHECK-NEXT: udivr z0.s, p0/m, z0.s, z1.s900; CHECK-NEXT: st1b { z0.s }, p0, [x0]901; CHECK-NEXT: ret902 %op1 = load <32 x i8>, ptr %a903 %op2 = load <32 x i8>, ptr %b904 %res = udiv <32 x i8> %op1, %op2905 store <32 x i8> %res, ptr %a906 ret void907}908 909define void @udiv_v64i8(ptr %a, ptr %b) vscale_range(16,0) #0 {910; CHECK-LABEL: udiv_v64i8:911; CHECK: // %bb.0:912; CHECK-NEXT: ptrue p0.s, vl64913; CHECK-NEXT: ld1b { z0.s }, p0/z, [x1]914; CHECK-NEXT: ld1b { z1.s }, p0/z, [x0]915; CHECK-NEXT: udivr z0.s, p0/m, z0.s, z1.s916; CHECK-NEXT: st1b { z0.s }, p0, [x0]917; CHECK-NEXT: ret918 %op1 = load <64 x i8>, ptr %a919 %op2 = load <64 x i8>, ptr %b920 %res = udiv <64 x i8> %op1, %op2921 store <64 x i8> %res, ptr %a922 ret void923}924 925define void @udiv_v128i8(ptr %a, ptr %b) vscale_range(16,0) #0 {926; CHECK-LABEL: udiv_v128i8:927; CHECK: // %bb.0:928; CHECK-NEXT: ptrue p0.h, vl128929; CHECK-NEXT: ptrue p1.s, vl64930; CHECK-NEXT: ld1b { z0.h }, p0/z, [x1]931; CHECK-NEXT: ld1b { z1.h }, p0/z, [x0]932; CHECK-NEXT: uunpklo z2.s, z0.h933; CHECK-NEXT: uunpklo z3.s, z1.h934; CHECK-NEXT: ext z0.b, z0.b, z0.b, #128935; CHECK-NEXT: ext z1.b, z1.b, z1.b, #128936; CHECK-NEXT: uunpklo z0.s, z0.h937; CHECK-NEXT: uunpklo z1.s, z1.h938; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s939; CHECK-NEXT: udivr z0.s, p1/m, z0.s, z1.s940; CHECK-NEXT: ptrue p1.h, vl64941; CHECK-NEXT: uzp1 z1.h, z2.h, z2.h942; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h943; CHECK-NEXT: splice z1.h, p1, z1.h, z0.h944; CHECK-NEXT: st1b { z1.h }, p0, [x0]945; CHECK-NEXT: ret946 %op1 = load <128 x i8>, ptr %a947 %op2 = load <128 x i8>, ptr %b948 %res = udiv <128 x i8> %op1, %op2949 store <128 x i8> %res, ptr %a950 ret void951}952 953define void @udiv_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {954; CHECK-LABEL: udiv_v256i8:955; CHECK: // %bb.0:956; CHECK-NEXT: ptrue p0.b, vl256957; CHECK-NEXT: ptrue p1.s, vl64958; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]959; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]960; CHECK-NEXT: uunpklo z2.h, z1.b961; CHECK-NEXT: uunpklo z3.h, z0.b962; CHECK-NEXT: ext z1.b, z1.b, z1.b, #128963; CHECK-NEXT: ext z0.b, z0.b, z0.b, #128964; CHECK-NEXT: uunpklo z1.h, z1.b965; CHECK-NEXT: uunpklo z4.s, z2.h966; CHECK-NEXT: uunpklo z5.s, z3.h967; CHECK-NEXT: ext z2.b, z2.b, z2.b, #128968; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128969; CHECK-NEXT: uunpklo z0.h, z0.b970; CHECK-NEXT: uunpklo z2.s, z2.h971; CHECK-NEXT: uunpklo z3.s, z3.h972; CHECK-NEXT: udivr z4.s, p1/m, z4.s, z5.s973; CHECK-NEXT: uunpklo z5.s, z0.h974; CHECK-NEXT: ext z0.b, z0.b, z0.b, #128975; CHECK-NEXT: uunpklo z0.s, z0.h976; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s977; CHECK-NEXT: uunpklo z3.s, z1.h978; CHECK-NEXT: ext z1.b, z1.b, z1.b, #128979; CHECK-NEXT: uunpklo z1.s, z1.h980; CHECK-NEXT: udivr z3.s, p1/m, z3.s, z5.s981; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h982; CHECK-NEXT: udiv z0.s, p1/m, z0.s, z1.s983; CHECK-NEXT: uzp1 z1.h, z4.h, z4.h984; CHECK-NEXT: ptrue p1.h, vl64985; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h986; CHECK-NEXT: splice z1.h, p1, z1.h, z2.h987; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h988; CHECK-NEXT: splice z3.h, p1, z3.h, z0.h989; CHECK-NEXT: uzp1 z0.b, z1.b, z1.b990; CHECK-NEXT: ptrue p1.b, vl128991; CHECK-NEXT: uzp1 z1.b, z3.b, z3.b992; CHECK-NEXT: splice z0.b, p1, z0.b, z1.b993; CHECK-NEXT: st1b { z0.b }, p0, [x0]994; CHECK-NEXT: ret995 %op1 = load <256 x i8>, ptr %a996 %op2 = load <256 x i8>, ptr %b997 %res = udiv <256 x i8> %op1, %op2998 store <256 x i8> %res, ptr %a999 ret void1000}1001 1002; Vector vXi16 udiv are not legal for NEON so use SVE when available.1003; FIXME: We should be able to improve the codegen for >= 256 bits here.1004define <4 x i16> @udiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 {1005; VBITS_GE_128-LABEL: udiv_v4i16:1006; VBITS_GE_128: // %bb.0:1007; VBITS_GE_128-NEXT: ushll v1.4s, v1.4h, #01008; VBITS_GE_128-NEXT: ushll v0.4s, v0.4h, #01009; VBITS_GE_128-NEXT: ptrue p0.s, vl41010; VBITS_GE_128-NEXT: udiv z0.s, p0/m, z0.s, z1.s1011; VBITS_GE_128-NEXT: xtn v0.4h, v0.4s1012; VBITS_GE_128-NEXT: ret1013;1014; VBITS_GE_256-LABEL: udiv_v4i16:1015; VBITS_GE_256: // %bb.0:1016; VBITS_GE_256-NEXT: ushll v1.4s, v1.4h, #01017; VBITS_GE_256-NEXT: ushll v0.4s, v0.4h, #01018; VBITS_GE_256-NEXT: ptrue p0.s, vl41019; VBITS_GE_256-NEXT: udivr z1.s, p0/m, z1.s, z0.s1020; VBITS_GE_256-NEXT: mov w8, v1.s[1]1021; VBITS_GE_256-NEXT: mov v0.16b, v1.16b1022; VBITS_GE_256-NEXT: mov w9, v1.s[2]1023; VBITS_GE_256-NEXT: mov v0.h[1], w81024; VBITS_GE_256-NEXT: mov w8, v1.s[3]1025; VBITS_GE_256-NEXT: mov v0.h[2], w91026; VBITS_GE_256-NEXT: mov v0.h[3], w81027; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 killed $q01028; VBITS_GE_256-NEXT: ret1029;1030; VBITS_GE_512-LABEL: udiv_v4i16:1031; VBITS_GE_512: // %bb.0:1032; VBITS_GE_512-NEXT: ushll v1.4s, v1.4h, #01033; VBITS_GE_512-NEXT: ushll v0.4s, v0.4h, #01034; VBITS_GE_512-NEXT: ptrue p0.s, vl41035; VBITS_GE_512-NEXT: udivr z1.s, p0/m, z1.s, z0.s1036; VBITS_GE_512-NEXT: mov w8, v1.s[1]1037; VBITS_GE_512-NEXT: mov v0.16b, v1.16b1038; VBITS_GE_512-NEXT: mov w9, v1.s[2]1039; VBITS_GE_512-NEXT: mov v0.h[1], w81040; VBITS_GE_512-NEXT: mov w8, v1.s[3]1041; VBITS_GE_512-NEXT: mov v0.h[2], w91042; VBITS_GE_512-NEXT: mov v0.h[3], w81043; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 killed $q01044; VBITS_GE_512-NEXT: ret1045 %res = udiv <4 x i16> %op1, %op21046 ret <4 x i16> %res1047}1048 1049define <8 x i16> @udiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 {1050; VBITS_GE_128-LABEL: udiv_v8i16:1051; VBITS_GE_128: // %bb.0:1052; VBITS_GE_128-NEXT: ushll2 v2.4s, v1.8h, #01053; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #01054; VBITS_GE_128-NEXT: ushll v1.4s, v1.4h, #01055; VBITS_GE_128-NEXT: ushll v0.4s, v0.4h, #01056; VBITS_GE_128-NEXT: ptrue p0.s, vl41057; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s1058; VBITS_GE_128-NEXT: udiv z0.s, p0/m, z0.s, z1.s1059; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v2.8h1060; VBITS_GE_128-NEXT: ret1061;1062; VBITS_GE_256-LABEL: udiv_v8i16:1063; VBITS_GE_256: // %bb.0:1064; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z11065; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z01066; VBITS_GE_256-NEXT: ptrue p0.s, vl81067; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h1068; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h1069; VBITS_GE_256-NEXT: udiv z0.s, p0/m, z0.s, z1.s1070; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h1071; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z01072; VBITS_GE_256-NEXT: ret1073;1074; VBITS_GE_512-LABEL: udiv_v8i16:1075; VBITS_GE_512: // %bb.0:1076; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z11077; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z01078; VBITS_GE_512-NEXT: ptrue p0.s, vl81079; VBITS_GE_512-NEXT: uunpklo z1.s, z1.h1080; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h1081; VBITS_GE_512-NEXT: udiv z0.s, p0/m, z0.s, z1.s1082; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h1083; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z01084; VBITS_GE_512-NEXT: ret1085 %res = udiv <8 x i16> %op1, %op21086 ret <8 x i16> %res1087}1088 1089define void @udiv_v16i16(ptr %a, ptr %b) #0 {1090; VBITS_GE_128-LABEL: udiv_v16i16:1091; VBITS_GE_128: // %bb.0:1092; VBITS_GE_128-NEXT: ldp q4, q1, [x1]1093; VBITS_GE_128-NEXT: ptrue p0.s, vl41094; VBITS_GE_128-NEXT: ldr q0, [x0, #16]1095; VBITS_GE_128-NEXT: ushll2 v2.4s, v1.8h, #01096; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #01097; VBITS_GE_128-NEXT: ushll2 v5.4s, v4.8h, #01098; VBITS_GE_128-NEXT: ushll v4.4s, v4.4h, #01099; VBITS_GE_128-NEXT: ushll v1.4s, v1.4h, #01100; VBITS_GE_128-NEXT: ushll v0.4s, v0.4h, #01101; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s1102; VBITS_GE_128-NEXT: ldr q3, [x0]1103; VBITS_GE_128-NEXT: ushll2 v6.4s, v3.8h, #01104; VBITS_GE_128-NEXT: ushll v3.4s, v3.4h, #01105; VBITS_GE_128-NEXT: udivr z5.s, p0/m, z5.s, z6.s1106; VBITS_GE_128-NEXT: udiv z3.s, p0/m, z3.s, z4.s1107; VBITS_GE_128-NEXT: udiv z0.s, p0/m, z0.s, z1.s1108; VBITS_GE_128-NEXT: uzp1 v1.8h, v3.8h, v5.8h1109; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v2.8h1110; VBITS_GE_128-NEXT: stp q1, q0, [x0]1111; VBITS_GE_128-NEXT: ret1112;1113; VBITS_GE_256-LABEL: udiv_v16i16:1114; VBITS_GE_256: // %bb.0:1115; VBITS_GE_256-NEXT: ptrue p0.h, vl161116; VBITS_GE_256-NEXT: ptrue p1.s, vl81117; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]1118; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1]1119; VBITS_GE_256-NEXT: uunpklo z2.s, z1.h1120; VBITS_GE_256-NEXT: uunpklo z3.s, z0.h1121; VBITS_GE_256-NEXT: ext z1.b, z1.b, z1.b, #161122; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #161123; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h1124; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h1125; VBITS_GE_256-NEXT: udivr z2.s, p1/m, z2.s, z3.s1126; VBITS_GE_256-NEXT: udiv z0.s, p1/m, z0.s, z1.s1127; VBITS_GE_256-NEXT: ptrue p1.h, vl81128; VBITS_GE_256-NEXT: uzp1 z1.h, z2.h, z2.h1129; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h1130; VBITS_GE_256-NEXT: splice z1.h, p1, z1.h, z0.h1131; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]1132; VBITS_GE_256-NEXT: ret1133;1134; VBITS_GE_512-LABEL: udiv_v16i16:1135; VBITS_GE_512: // %bb.0:1136; VBITS_GE_512-NEXT: ptrue p0.s, vl161137; VBITS_GE_512-NEXT: ld1h { z0.s }, p0/z, [x1]1138; VBITS_GE_512-NEXT: ld1h { z1.s }, p0/z, [x0]1139; VBITS_GE_512-NEXT: udivr z0.s, p0/m, z0.s, z1.s1140; VBITS_GE_512-NEXT: st1h { z0.s }, p0, [x0]1141; VBITS_GE_512-NEXT: ret1142 %op1 = load <16 x i16>, ptr %a1143 %op2 = load <16 x i16>, ptr %b1144 %res = udiv <16 x i16> %op1, %op21145 store <16 x i16> %res, ptr %a1146 ret void1147}1148 1149define void @udiv_v32i16(ptr %a, ptr %b) vscale_range(8,0) #0 {1150; CHECK-LABEL: udiv_v32i16:1151; CHECK: // %bb.0:1152; CHECK-NEXT: ptrue p0.s, vl321153; CHECK-NEXT: ld1h { z0.s }, p0/z, [x1]1154; CHECK-NEXT: ld1h { z1.s }, p0/z, [x0]1155; CHECK-NEXT: udivr z0.s, p0/m, z0.s, z1.s1156; CHECK-NEXT: st1h { z0.s }, p0, [x0]1157; CHECK-NEXT: ret1158 %op1 = load <32 x i16>, ptr %a1159 %op2 = load <32 x i16>, ptr %b1160 %res = udiv <32 x i16> %op1, %op21161 store <32 x i16> %res, ptr %a1162 ret void1163}1164 1165define void @udiv_v64i16(ptr %a, ptr %b) vscale_range(16,0) #0 {1166; CHECK-LABEL: udiv_v64i16:1167; CHECK: // %bb.0:1168; CHECK-NEXT: ptrue p0.s, vl641169; CHECK-NEXT: ld1h { z0.s }, p0/z, [x1]1170; CHECK-NEXT: ld1h { z1.s }, p0/z, [x0]1171; CHECK-NEXT: udivr z0.s, p0/m, z0.s, z1.s1172; CHECK-NEXT: st1h { z0.s }, p0, [x0]1173; CHECK-NEXT: ret1174 %op1 = load <64 x i16>, ptr %a1175 %op2 = load <64 x i16>, ptr %b1176 %res = udiv <64 x i16> %op1, %op21177 store <64 x i16> %res, ptr %a1178 ret void1179}1180 1181define void @udiv_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {1182; CHECK-LABEL: udiv_v128i16:1183; CHECK: // %bb.0:1184; CHECK-NEXT: ptrue p0.h, vl1281185; CHECK-NEXT: ptrue p1.s, vl641186; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1187; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]1188; CHECK-NEXT: uunpklo z2.s, z1.h1189; CHECK-NEXT: uunpklo z3.s, z0.h1190; CHECK-NEXT: ext z1.b, z1.b, z1.b, #1281191; CHECK-NEXT: ext z0.b, z0.b, z0.b, #1281192; CHECK-NEXT: uunpklo z1.s, z1.h1193; CHECK-NEXT: uunpklo z0.s, z0.h1194; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s1195; CHECK-NEXT: udiv z0.s, p1/m, z0.s, z1.s1196; CHECK-NEXT: ptrue p1.h, vl641197; CHECK-NEXT: uzp1 z1.h, z2.h, z2.h1198; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h1199; CHECK-NEXT: splice z1.h, p1, z1.h, z0.h1200; CHECK-NEXT: st1h { z1.h }, p0, [x0]1201; CHECK-NEXT: ret1202 %op1 = load <128 x i16>, ptr %a1203 %op2 = load <128 x i16>, ptr %b1204 %res = udiv <128 x i16> %op1, %op21205 store <128 x i16> %res, ptr %a1206 ret void1207}1208 1209; Vector v2i32 udiv are not legal for NEON so use SVE when available.1210define <2 x i32> @udiv_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(1,0) #0 {1211; CHECK-LABEL: udiv_v2i32:1212; CHECK: // %bb.0:1213; CHECK-NEXT: ptrue p0.s, vl21214; CHECK-NEXT: // kill: def $d0 killed $d0 def $z01215; CHECK-NEXT: // kill: def $d1 killed $d1 def $z11216; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s1217; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z01218; CHECK-NEXT: ret1219 %res = udiv <2 x i32> %op1, %op21220 ret <2 x i32> %res1221}1222 1223; Vector v4i32 udiv are not legal for NEON so use SVE when available.1224define <4 x i32> @udiv_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(1,0) #0 {1225; CHECK-LABEL: udiv_v4i32:1226; CHECK: // %bb.0:1227; CHECK-NEXT: ptrue p0.s, vl41228; CHECK-NEXT: // kill: def $q0 killed $q0 def $z01229; CHECK-NEXT: // kill: def $q1 killed $q1 def $z11230; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s1231; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z01232; CHECK-NEXT: ret1233 %res = udiv <4 x i32> %op1, %op21234 ret <4 x i32> %res1235}1236 1237define void @udiv_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {1238; CHECK-LABEL: udiv_v8i32:1239; CHECK: // %bb.0:1240; CHECK-NEXT: ptrue p0.s, vl81241; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1242; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]1243; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s1244; CHECK-NEXT: st1w { z0.s }, p0, [x0]1245; CHECK-NEXT: ret1246 %op1 = load <8 x i32>, ptr %a1247 %op2 = load <8 x i32>, ptr %b1248 %res = udiv <8 x i32> %op1, %op21249 store <8 x i32> %res, ptr %a1250 ret void1251}1252 1253define void @udiv_v16i32(ptr %a, ptr %b) #0 {1254; VBITS_GE_128-LABEL: udiv_v16i32:1255; VBITS_GE_128: // %bb.0:1256; VBITS_GE_128-NEXT: ldp q0, q3, [x1]1257; VBITS_GE_128-NEXT: ptrue p0.s, vl41258; VBITS_GE_128-NEXT: ldp q1, q2, [x0]1259; VBITS_GE_128-NEXT: ldp q5, q4, [x1, #32]1260; VBITS_GE_128-NEXT: udivr z0.s, p0/m, z0.s, z1.s1261; VBITS_GE_128-NEXT: ldr q1, [x0, #48]1262; VBITS_GE_128-NEXT: udiv z1.s, p0/m, z1.s, z4.s1263; VBITS_GE_128-NEXT: ldr q4, [x0, #32]1264; VBITS_GE_128-NEXT: udiv z4.s, p0/m, z4.s, z5.s1265; VBITS_GE_128-NEXT: udiv z2.s, p0/m, z2.s, z3.s1266; VBITS_GE_128-NEXT: stp q4, q1, [x0, #32]1267; VBITS_GE_128-NEXT: stp q0, q2, [x0]1268; VBITS_GE_128-NEXT: ret1269;1270; VBITS_GE_256-LABEL: udiv_v16i32:1271; VBITS_GE_256: // %bb.0:1272; VBITS_GE_256-NEXT: ptrue p0.s, vl81273; VBITS_GE_256-NEXT: mov x8, #8 // =0x81274; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]1275; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]1276; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x1]1277; VBITS_GE_256-NEXT: udiv z0.s, p0/m, z0.s, z1.s1278; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]1279; VBITS_GE_256-NEXT: udiv z1.s, p0/m, z1.s, z2.s1280; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]1281; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]1282; VBITS_GE_256-NEXT: ret1283;1284; VBITS_GE_512-LABEL: udiv_v16i32:1285; VBITS_GE_512: // %bb.0:1286; VBITS_GE_512-NEXT: ptrue p0.s, vl161287; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]1288; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]1289; VBITS_GE_512-NEXT: udiv z0.s, p0/m, z0.s, z1.s1290; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]1291; VBITS_GE_512-NEXT: ret1292 %op1 = load <16 x i32>, ptr %a1293 %op2 = load <16 x i32>, ptr %b1294 %res = udiv <16 x i32> %op1, %op21295 store <16 x i32> %res, ptr %a1296 ret void1297}1298 1299define void @udiv_v32i32(ptr %a, ptr %b) vscale_range(8,0) #0 {1300; CHECK-LABEL: udiv_v32i32:1301; CHECK: // %bb.0:1302; CHECK-NEXT: ptrue p0.s, vl321303; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1304; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]1305; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s1306; CHECK-NEXT: st1w { z0.s }, p0, [x0]1307; CHECK-NEXT: ret1308 %op1 = load <32 x i32>, ptr %a1309 %op2 = load <32 x i32>, ptr %b1310 %res = udiv <32 x i32> %op1, %op21311 store <32 x i32> %res, ptr %a1312 ret void1313}1314 1315define void @udiv_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {1316; CHECK-LABEL: udiv_v64i32:1317; CHECK: // %bb.0:1318; CHECK-NEXT: ptrue p0.s, vl641319; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1320; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]1321; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s1322; CHECK-NEXT: st1w { z0.s }, p0, [x0]1323; CHECK-NEXT: ret1324 %op1 = load <64 x i32>, ptr %a1325 %op2 = load <64 x i32>, ptr %b1326 %res = udiv <64 x i32> %op1, %op21327 store <64 x i32> %res, ptr %a1328 ret void1329}1330 1331; Vector i64 udiv are not legal for NEON so use SVE when available.1332define <1 x i64> @udiv_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(1,0) #0 {1333; CHECK-LABEL: udiv_v1i64:1334; CHECK: // %bb.0:1335; CHECK-NEXT: ptrue p0.d, vl11336; CHECK-NEXT: // kill: def $d0 killed $d0 def $z01337; CHECK-NEXT: // kill: def $d1 killed $d1 def $z11338; CHECK-NEXT: udiv z0.d, p0/m, z0.d, z1.d1339; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z01340; CHECK-NEXT: ret1341 %res = udiv <1 x i64> %op1, %op21342 ret <1 x i64> %res1343}1344 1345; Vector i64 udiv are not legal for NEON so use SVE when available.1346define <2 x i64> @udiv_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(1,0) #0 {1347; CHECK-LABEL: udiv_v2i64:1348; CHECK: // %bb.0:1349; CHECK-NEXT: ptrue p0.d, vl21350; CHECK-NEXT: // kill: def $q0 killed $q0 def $z01351; CHECK-NEXT: // kill: def $q1 killed $q1 def $z11352; CHECK-NEXT: udiv z0.d, p0/m, z0.d, z1.d1353; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z01354; CHECK-NEXT: ret1355 %res = udiv <2 x i64> %op1, %op21356 ret <2 x i64> %res1357}1358 1359define void @udiv_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {1360; CHECK-LABEL: udiv_v4i64:1361; CHECK: // %bb.0:1362; CHECK-NEXT: ptrue p0.d, vl41363; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1364; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]1365; CHECK-NEXT: udiv z0.d, p0/m, z0.d, z1.d1366; CHECK-NEXT: st1d { z0.d }, p0, [x0]1367; CHECK-NEXT: ret1368 %op1 = load <4 x i64>, ptr %a1369 %op2 = load <4 x i64>, ptr %b1370 %res = udiv <4 x i64> %op1, %op21371 store <4 x i64> %res, ptr %a1372 ret void1373}1374 1375define void @udiv_v8i64(ptr %a, ptr %b) #0 {1376; VBITS_GE_128-LABEL: udiv_v8i64:1377; VBITS_GE_128: // %bb.0:1378; VBITS_GE_128-NEXT: ldp q0, q3, [x1]1379; VBITS_GE_128-NEXT: ptrue p0.d, vl21380; VBITS_GE_128-NEXT: ldp q1, q2, [x0]1381; VBITS_GE_128-NEXT: ldp q5, q4, [x1, #32]1382; VBITS_GE_128-NEXT: udivr z0.d, p0/m, z0.d, z1.d1383; VBITS_GE_128-NEXT: ldr q1, [x0, #48]1384; VBITS_GE_128-NEXT: udiv z1.d, p0/m, z1.d, z4.d1385; VBITS_GE_128-NEXT: ldr q4, [x0, #32]1386; VBITS_GE_128-NEXT: udiv z4.d, p0/m, z4.d, z5.d1387; VBITS_GE_128-NEXT: udiv z2.d, p0/m, z2.d, z3.d1388; VBITS_GE_128-NEXT: stp q4, q1, [x0, #32]1389; VBITS_GE_128-NEXT: stp q0, q2, [x0]1390; VBITS_GE_128-NEXT: ret1391;1392; VBITS_GE_256-LABEL: udiv_v8i64:1393; VBITS_GE_256: // %bb.0:1394; VBITS_GE_256-NEXT: ptrue p0.d, vl41395; VBITS_GE_256-NEXT: mov x8, #4 // =0x41396; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]1397; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]1398; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1]1399; VBITS_GE_256-NEXT: udiv z0.d, p0/m, z0.d, z1.d1400; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]1401; VBITS_GE_256-NEXT: udiv z1.d, p0/m, z1.d, z2.d1402; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]1403; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]1404; VBITS_GE_256-NEXT: ret1405;1406; VBITS_GE_512-LABEL: udiv_v8i64:1407; VBITS_GE_512: // %bb.0:1408; VBITS_GE_512-NEXT: ptrue p0.d, vl81409; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]1410; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]1411; VBITS_GE_512-NEXT: udiv z0.d, p0/m, z0.d, z1.d1412; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]1413; VBITS_GE_512-NEXT: ret1414 %op1 = load <8 x i64>, ptr %a1415 %op2 = load <8 x i64>, ptr %b1416 %res = udiv <8 x i64> %op1, %op21417 store <8 x i64> %res, ptr %a1418 ret void1419}1420 1421define void @udiv_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 {1422; CHECK-LABEL: udiv_v16i64:1423; CHECK: // %bb.0:1424; CHECK-NEXT: ptrue p0.d, vl161425; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1426; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]1427; CHECK-NEXT: udiv z0.d, p0/m, z0.d, z1.d1428; CHECK-NEXT: st1d { z0.d }, p0, [x0]1429; CHECK-NEXT: ret1430 %op1 = load <16 x i64>, ptr %a1431 %op2 = load <16 x i64>, ptr %b1432 %res = udiv <16 x i64> %op1, %op21433 store <16 x i64> %res, ptr %a1434 ret void1435}1436 1437define void @udiv_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {1438; CHECK-LABEL: udiv_v32i64:1439; CHECK: // %bb.0:1440; CHECK-NEXT: ptrue p0.d, vl321441; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1442; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]1443; CHECK-NEXT: udiv z0.d, p0/m, z0.d, z1.d1444; CHECK-NEXT: st1d { z0.d }, p0, [x0]1445; CHECK-NEXT: ret1446 %op1 = load <32 x i64>, ptr %a1447 %op2 = load <32 x i64>, ptr %b1448 %res = udiv <32 x i64> %op1, %op21449 store <32 x i64> %res, ptr %a1450 ret void1451}1452 1453; This used to crash because isUnaryPredicate and BuildUDIV don't know how1454; a SPLAT_VECTOR of fixed vector type should be handled.1455define void @udiv_constantsplat_v8i32(ptr %a) vscale_range(2,0) #1 {1456; CHECK-LABEL: udiv_constantsplat_v8i32:1457; CHECK: // %bb.0:1458; CHECK-NEXT: ptrue p0.s, vl81459; CHECK-NEXT: mov z1.s, #95 // =0x5f1460; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1461; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s1462; CHECK-NEXT: st1w { z0.s }, p0, [x0]1463; CHECK-NEXT: ret1464 %op1 = load <8 x i32>, ptr %a1465 %res = udiv <8 x i32> %op1, <i32 95, i32 95, i32 95, i32 95, i32 95, i32 95, i32 95, i32 95>1466 store <8 x i32> %res, ptr %a1467 ret void1468}1469 1470attributes #0 = { "target-features"="+sve" }1471attributes #1 = { "target-features"="+sve" minsize }1472