1640 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -aarch64-sve-vector-bits-min=128 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_1283; RUN: llc -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2564; RUN: llc -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5125; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5126 7target triple = "aarch64-unknown-linux-gnu"8 9;10; SREM11;12 13; Vector vXi8 sdiv are not legal for NEON so use SVE when available.14; FIXME: We should be able to improve the codegen for >= 256 bits here.15define <8 x i8> @srem_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 {16; VBITS_GE_128-LABEL: srem_v8i8:17; VBITS_GE_128: // %bb.0:18; VBITS_GE_128-NEXT: sshll v2.8h, v1.8b, #019; VBITS_GE_128-NEXT: sshll v3.8h, v0.8b, #020; VBITS_GE_128-NEXT: ptrue p0.s, vl421; VBITS_GE_128-NEXT: sshll2 v4.4s, v2.8h, #022; VBITS_GE_128-NEXT: sshll2 v5.4s, v3.8h, #023; VBITS_GE_128-NEXT: sshll v2.4s, v2.4h, #024; VBITS_GE_128-NEXT: sshll v3.4s, v3.4h, #025; VBITS_GE_128-NEXT: sdivr z4.s, p0/m, z4.s, z5.s26; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s27; VBITS_GE_128-NEXT: uzp1 v2.8h, v2.8h, v4.8h28; VBITS_GE_128-NEXT: xtn v2.8b, v2.8h29; VBITS_GE_128-NEXT: mls v0.8b, v2.8b, v1.8b30; VBITS_GE_128-NEXT: ret31;32; VBITS_GE_256-LABEL: srem_v8i8:33; VBITS_GE_256: // %bb.0:34; VBITS_GE_256-NEXT: // kill: def $d1 killed $d1 def $z135; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 def $z036; VBITS_GE_256-NEXT: sunpklo z2.h, z1.b37; VBITS_GE_256-NEXT: sunpklo z3.h, z0.b38; VBITS_GE_256-NEXT: ptrue p0.s, vl839; VBITS_GE_256-NEXT: sunpklo z2.s, z2.h40; VBITS_GE_256-NEXT: sunpklo z3.s, z3.h41; VBITS_GE_256-NEXT: sdivr z2.s, p0/m, z2.s, z3.s42; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h43; VBITS_GE_256-NEXT: umov w8, v2.h[0]44; VBITS_GE_256-NEXT: umov w9, v2.h[1]45; VBITS_GE_256-NEXT: fmov s3, w846; VBITS_GE_256-NEXT: umov w8, v2.h[2]47; VBITS_GE_256-NEXT: mov v3.b[1], w948; VBITS_GE_256-NEXT: mov v3.b[2], w849; VBITS_GE_256-NEXT: umov w8, v2.h[3]50; VBITS_GE_256-NEXT: mov v3.b[3], w851; VBITS_GE_256-NEXT: umov w8, v2.h[4]52; VBITS_GE_256-NEXT: mov v3.b[4], w853; VBITS_GE_256-NEXT: umov w8, v2.h[5]54; VBITS_GE_256-NEXT: mov v3.b[5], w855; VBITS_GE_256-NEXT: umov w8, v2.h[6]56; VBITS_GE_256-NEXT: mov v3.b[6], w857; VBITS_GE_256-NEXT: umov w8, v2.h[7]58; VBITS_GE_256-NEXT: mov v3.b[7], w859; VBITS_GE_256-NEXT: mls v0.8b, v3.8b, v1.8b60; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 killed $z061; VBITS_GE_256-NEXT: ret62;63; VBITS_GE_512-LABEL: srem_v8i8:64; VBITS_GE_512: // %bb.0:65; VBITS_GE_512-NEXT: // kill: def $d1 killed $d1 def $z166; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 def $z067; VBITS_GE_512-NEXT: sunpklo z2.h, z1.b68; VBITS_GE_512-NEXT: sunpklo z3.h, z0.b69; VBITS_GE_512-NEXT: ptrue p0.s, vl870; VBITS_GE_512-NEXT: sunpklo z2.s, z2.h71; VBITS_GE_512-NEXT: sunpklo z3.s, z3.h72; VBITS_GE_512-NEXT: sdivr z2.s, p0/m, z2.s, z3.s73; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h74; VBITS_GE_512-NEXT: umov w8, v2.h[0]75; VBITS_GE_512-NEXT: umov w9, v2.h[1]76; VBITS_GE_512-NEXT: fmov s3, w877; VBITS_GE_512-NEXT: umov w8, v2.h[2]78; VBITS_GE_512-NEXT: mov v3.b[1], w979; VBITS_GE_512-NEXT: mov v3.b[2], w880; VBITS_GE_512-NEXT: umov w8, v2.h[3]81; VBITS_GE_512-NEXT: mov v3.b[3], w882; VBITS_GE_512-NEXT: umov w8, v2.h[4]83; VBITS_GE_512-NEXT: mov v3.b[4], w884; VBITS_GE_512-NEXT: umov w8, v2.h[5]85; VBITS_GE_512-NEXT: mov v3.b[5], w886; VBITS_GE_512-NEXT: umov w8, v2.h[6]87; VBITS_GE_512-NEXT: mov v3.b[6], w888; VBITS_GE_512-NEXT: umov w8, v2.h[7]89; VBITS_GE_512-NEXT: mov v3.b[7], w890; VBITS_GE_512-NEXT: mls v0.8b, v3.8b, v1.8b91; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 killed $z092; VBITS_GE_512-NEXT: ret93 %res = srem <8 x i8> %op1, %op294 ret <8 x i8> %res95}96 97define <16 x i8> @srem_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 {98; VBITS_GE_128-LABEL: srem_v16i8:99; VBITS_GE_128: // %bb.0:100; VBITS_GE_128-NEXT: sshll2 v2.8h, v1.16b, #0101; VBITS_GE_128-NEXT: sshll2 v3.8h, v0.16b, #0102; VBITS_GE_128-NEXT: ptrue p0.s, vl4103; VBITS_GE_128-NEXT: sshll2 v4.4s, v2.8h, #0104; VBITS_GE_128-NEXT: sshll2 v5.4s, v3.8h, #0105; VBITS_GE_128-NEXT: sshll v2.4s, v2.4h, #0106; VBITS_GE_128-NEXT: sshll v3.4s, v3.4h, #0107; VBITS_GE_128-NEXT: sdivr z4.s, p0/m, z4.s, z5.s108; VBITS_GE_128-NEXT: sshll v5.8h, v0.8b, #0109; VBITS_GE_128-NEXT: sshll2 v7.4s, v5.8h, #0110; VBITS_GE_128-NEXT: sshll v5.4s, v5.4h, #0111; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s112; VBITS_GE_128-NEXT: sshll v3.8h, v1.8b, #0113; VBITS_GE_128-NEXT: sshll2 v6.4s, v3.8h, #0114; VBITS_GE_128-NEXT: sshll v3.4s, v3.4h, #0115; VBITS_GE_128-NEXT: sdivr z6.s, p0/m, z6.s, z7.s116; VBITS_GE_128-NEXT: uzp1 v2.8h, v2.8h, v4.8h117; VBITS_GE_128-NEXT: sdivr z3.s, p0/m, z3.s, z5.s118; VBITS_GE_128-NEXT: uzp1 v3.8h, v3.8h, v6.8h119; VBITS_GE_128-NEXT: uzp1 v2.16b, v3.16b, v2.16b120; VBITS_GE_128-NEXT: mls v0.16b, v2.16b, v1.16b121; VBITS_GE_128-NEXT: ret122;123; VBITS_GE_256-LABEL: srem_v16i8:124; VBITS_GE_256: // %bb.0:125; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1126; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0127; VBITS_GE_256-NEXT: sunpklo z2.h, z1.b128; VBITS_GE_256-NEXT: sunpklo z3.h, z0.b129; VBITS_GE_256-NEXT: ptrue p0.s, vl8130; VBITS_GE_256-NEXT: sunpklo z4.s, z2.h131; VBITS_GE_256-NEXT: sunpklo z5.s, z3.h132; VBITS_GE_256-NEXT: ext z2.b, z2.b, z2.b, #16133; VBITS_GE_256-NEXT: ext z3.b, z3.b, z3.b, #16134; VBITS_GE_256-NEXT: sunpklo z2.s, z2.h135; VBITS_GE_256-NEXT: sunpklo z3.s, z3.h136; VBITS_GE_256-NEXT: sdivr z4.s, p0/m, z4.s, z5.s137; VBITS_GE_256-NEXT: sdivr z2.s, p0/m, z2.s, z3.s138; VBITS_GE_256-NEXT: ptrue p0.h, vl8139; VBITS_GE_256-NEXT: uzp1 z3.h, z4.h, z4.h140; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h141; VBITS_GE_256-NEXT: splice z3.h, p0, z3.h, z2.h142; VBITS_GE_256-NEXT: uzp1 z2.b, z3.b, z3.b143; VBITS_GE_256-NEXT: mls v0.16b, v2.16b, v1.16b144; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0145; VBITS_GE_256-NEXT: ret146;147; VBITS_GE_512-LABEL: srem_v16i8:148; VBITS_GE_512: // %bb.0:149; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1150; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0151; VBITS_GE_512-NEXT: sunpklo z2.h, z1.b152; VBITS_GE_512-NEXT: sunpklo z3.h, z0.b153; VBITS_GE_512-NEXT: ptrue p0.s, vl16154; VBITS_GE_512-NEXT: sunpklo z2.s, z2.h155; VBITS_GE_512-NEXT: sunpklo z3.s, z3.h156; VBITS_GE_512-NEXT: sdivr z2.s, p0/m, z2.s, z3.s157; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h158; VBITS_GE_512-NEXT: uzp1 z2.b, z2.b, z2.b159; VBITS_GE_512-NEXT: mls v0.16b, v2.16b, v1.16b160; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0161; VBITS_GE_512-NEXT: ret162 %res = srem <16 x i8> %op1, %op2163 ret <16 x i8> %res164}165 166define void @srem_v32i8(ptr %a, ptr %b) vscale_range(8,0) #0 {167; CHECK-LABEL: srem_v32i8:168; CHECK: // %bb.0:169; CHECK-NEXT: ptrue p0.b, vl32170; CHECK-NEXT: ptrue p1.s, vl32171; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]172; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]173; CHECK-NEXT: sunpklo z2.h, z1.b174; CHECK-NEXT: sunpklo z3.h, z0.b175; CHECK-NEXT: sunpklo z2.s, z2.h176; CHECK-NEXT: sunpklo z3.s, z3.h177; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s178; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h179; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b180; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b181; CHECK-NEXT: st1b { z0.b }, p0, [x0]182; CHECK-NEXT: ret183 %op1 = load <32 x i8>, ptr %a184 %op2 = load <32 x i8>, ptr %b185 %res = srem <32 x i8> %op1, %op2186 store <32 x i8> %res, ptr %a187 ret void188}189 190define void @srem_v64i8(ptr %a, ptr %b) vscale_range(16,0) #0 {191; CHECK-LABEL: srem_v64i8:192; CHECK: // %bb.0:193; CHECK-NEXT: ptrue p0.b, vl64194; CHECK-NEXT: ptrue p1.s, vl64195; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]196; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]197; CHECK-NEXT: sunpklo z2.h, z1.b198; CHECK-NEXT: sunpklo z3.h, z0.b199; CHECK-NEXT: sunpklo z2.s, z2.h200; CHECK-NEXT: sunpklo z3.s, z3.h201; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s202; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h203; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b204; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b205; CHECK-NEXT: st1b { z0.b }, p0, [x0]206; CHECK-NEXT: ret207 %op1 = load <64 x i8>, ptr %a208 %op2 = load <64 x i8>, ptr %b209 %res = srem <64 x i8> %op1, %op2210 store <64 x i8> %res, ptr %a211 ret void212}213 214define void @srem_v128i8(ptr %a, ptr %b) vscale_range(16,0) #0 {215; CHECK-LABEL: srem_v128i8:216; CHECK: // %bb.0:217; CHECK-NEXT: ptrue p0.b, vl128218; CHECK-NEXT: ptrue p1.s, vl64219; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]220; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]221; CHECK-NEXT: sunpklo z2.h, z1.b222; CHECK-NEXT: sunpklo z3.h, z0.b223; CHECK-NEXT: sunpklo z4.s, z2.h224; CHECK-NEXT: sunpklo z5.s, z3.h225; CHECK-NEXT: ext z2.b, z2.b, z2.b, #128226; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128227; CHECK-NEXT: sunpklo z2.s, z2.h228; CHECK-NEXT: sunpklo z3.s, z3.h229; CHECK-NEXT: sdivr z4.s, p1/m, z4.s, z5.s230; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s231; CHECK-NEXT: ptrue p1.h, vl64232; CHECK-NEXT: uzp1 z3.h, z4.h, z4.h233; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h234; CHECK-NEXT: splice z3.h, p1, z3.h, z2.h235; CHECK-NEXT: uzp1 z2.b, z3.b, z3.b236; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b237; CHECK-NEXT: st1b { z0.b }, p0, [x0]238; CHECK-NEXT: ret239 %op1 = load <128 x i8>, ptr %a240 %op2 = load <128 x i8>, ptr %b241 %res = srem <128 x i8> %op1, %op2242 store <128 x i8> %res, ptr %a243 ret void244}245 246define void @srem_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {247; CHECK-LABEL: srem_v256i8:248; CHECK: // %bb.0:249; CHECK-NEXT: ptrue p0.b, vl256250; CHECK-NEXT: ptrue p1.s, vl64251; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]252; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]253; CHECK-NEXT: sunpklo z2.h, z1.b254; CHECK-NEXT: sunpklo z3.h, z0.b255; CHECK-NEXT: sunpklo z4.s, z2.h256; CHECK-NEXT: sunpklo z5.s, z3.h257; CHECK-NEXT: ext z2.b, z2.b, z2.b, #128258; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128259; CHECK-NEXT: sunpklo z2.s, z2.h260; CHECK-NEXT: sunpklo z3.s, z3.h261; CHECK-NEXT: sdivr z4.s, p1/m, z4.s, z5.s262; CHECK-NEXT: movprfx z5, z0263; CHECK-NEXT: ext z5.b, z5.b, z0.b, #128264; CHECK-NEXT: sunpklo z5.h, z5.b265; CHECK-NEXT: sunpklo z7.s, z5.h266; CHECK-NEXT: ext z5.b, z5.b, z5.b, #128267; CHECK-NEXT: sunpklo z5.s, z5.h268; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s269; CHECK-NEXT: movprfx z3, z1270; CHECK-NEXT: ext z3.b, z3.b, z1.b, #128271; CHECK-NEXT: sunpklo z3.h, z3.b272; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h273; CHECK-NEXT: sunpklo z6.s, z3.h274; CHECK-NEXT: ext z3.b, z3.b, z3.b, #128275; CHECK-NEXT: sunpklo z3.s, z3.h276; CHECK-NEXT: sdivr z6.s, p1/m, z6.s, z7.s277; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h278; CHECK-NEXT: sdivr z3.s, p1/m, z3.s, z5.s279; CHECK-NEXT: ptrue p1.h, vl64280; CHECK-NEXT: splice z4.h, p1, z4.h, z2.h281; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h282; CHECK-NEXT: uzp1 z2.b, z4.b, z4.b283; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h284; CHECK-NEXT: splice z5.h, p1, z5.h, z3.h285; CHECK-NEXT: ptrue p1.b, vl128286; CHECK-NEXT: uzp1 z3.b, z5.b, z5.b287; CHECK-NEXT: splice z2.b, p1, z2.b, z3.b288; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b289; CHECK-NEXT: st1b { z0.b }, p0, [x0]290; CHECK-NEXT: ret291 %op1 = load <256 x i8>, ptr %a292 %op2 = load <256 x i8>, ptr %b293 %res = srem <256 x i8> %op1, %op2294 store <256 x i8> %res, ptr %a295 ret void296}297 298; Vector vXi16 sdiv are not legal for NEON so use SVE when available.299; FIXME: We should be able to improve the codegen for >= 256 bits here.300define <4 x i16> @srem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 {301; VBITS_GE_128-LABEL: srem_v4i16:302; VBITS_GE_128: // %bb.0:303; VBITS_GE_128-NEXT: sshll v2.4s, v1.4h, #0304; VBITS_GE_128-NEXT: sshll v3.4s, v0.4h, #0305; VBITS_GE_128-NEXT: ptrue p0.s, vl4306; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s307; VBITS_GE_128-NEXT: xtn v2.4h, v2.4s308; VBITS_GE_128-NEXT: mls v0.4h, v2.4h, v1.4h309; VBITS_GE_128-NEXT: ret310;311; VBITS_GE_256-LABEL: srem_v4i16:312; VBITS_GE_256: // %bb.0:313; VBITS_GE_256-NEXT: sshll v2.4s, v1.4h, #0314; VBITS_GE_256-NEXT: sshll v3.4s, v0.4h, #0315; VBITS_GE_256-NEXT: ptrue p0.s, vl4316; VBITS_GE_256-NEXT: sdivr z2.s, p0/m, z2.s, z3.s317; VBITS_GE_256-NEXT: mov w8, v2.s[1]318; VBITS_GE_256-NEXT: mov v3.16b, v2.16b319; VBITS_GE_256-NEXT: mov w9, v2.s[2]320; VBITS_GE_256-NEXT: mov v3.h[1], w8321; VBITS_GE_256-NEXT: mov w8, v2.s[3]322; VBITS_GE_256-NEXT: mov v3.h[2], w9323; VBITS_GE_256-NEXT: mov v3.h[3], w8324; VBITS_GE_256-NEXT: mls v0.4h, v3.4h, v1.4h325; VBITS_GE_256-NEXT: ret326;327; VBITS_GE_512-LABEL: srem_v4i16:328; VBITS_GE_512: // %bb.0:329; VBITS_GE_512-NEXT: sshll v2.4s, v1.4h, #0330; VBITS_GE_512-NEXT: sshll v3.4s, v0.4h, #0331; VBITS_GE_512-NEXT: ptrue p0.s, vl4332; VBITS_GE_512-NEXT: sdivr z2.s, p0/m, z2.s, z3.s333; VBITS_GE_512-NEXT: mov w8, v2.s[1]334; VBITS_GE_512-NEXT: mov v3.16b, v2.16b335; VBITS_GE_512-NEXT: mov w9, v2.s[2]336; VBITS_GE_512-NEXT: mov v3.h[1], w8337; VBITS_GE_512-NEXT: mov w8, v2.s[3]338; VBITS_GE_512-NEXT: mov v3.h[2], w9339; VBITS_GE_512-NEXT: mov v3.h[3], w8340; VBITS_GE_512-NEXT: mls v0.4h, v3.4h, v1.4h341; VBITS_GE_512-NEXT: ret342 %res = srem <4 x i16> %op1, %op2343 ret <4 x i16> %res344}345 346define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 {347; VBITS_GE_128-LABEL: srem_v8i16:348; VBITS_GE_128: // %bb.0:349; VBITS_GE_128-NEXT: sshll2 v2.4s, v1.8h, #0350; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0351; VBITS_GE_128-NEXT: ptrue p0.s, vl4352; VBITS_GE_128-NEXT: sshll v4.4s, v0.4h, #0353; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s354; VBITS_GE_128-NEXT: sshll v3.4s, v1.4h, #0355; VBITS_GE_128-NEXT: sdivr z3.s, p0/m, z3.s, z4.s356; VBITS_GE_128-NEXT: uzp1 v2.8h, v3.8h, v2.8h357; VBITS_GE_128-NEXT: mls v0.8h, v2.8h, v1.8h358; VBITS_GE_128-NEXT: ret359;360; VBITS_GE_256-LABEL: srem_v8i16:361; VBITS_GE_256: // %bb.0:362; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1363; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0364; VBITS_GE_256-NEXT: sunpklo z2.s, z1.h365; VBITS_GE_256-NEXT: sunpklo z3.s, z0.h366; VBITS_GE_256-NEXT: ptrue p0.s, vl8367; VBITS_GE_256-NEXT: sdivr z2.s, p0/m, z2.s, z3.s368; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h369; VBITS_GE_256-NEXT: mls v0.8h, v2.8h, v1.8h370; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0371; VBITS_GE_256-NEXT: ret372;373; VBITS_GE_512-LABEL: srem_v8i16:374; VBITS_GE_512: // %bb.0:375; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1376; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0377; VBITS_GE_512-NEXT: sunpklo z2.s, z1.h378; VBITS_GE_512-NEXT: sunpklo z3.s, z0.h379; VBITS_GE_512-NEXT: ptrue p0.s, vl8380; VBITS_GE_512-NEXT: sdivr z2.s, p0/m, z2.s, z3.s381; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h382; VBITS_GE_512-NEXT: mls v0.8h, v2.8h, v1.8h383; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0384; VBITS_GE_512-NEXT: ret385 %res = srem <8 x i16> %op1, %op2386 ret <8 x i16> %res387}388 389define void @srem_v16i16(ptr %a, ptr %b) #0 {390; VBITS_GE_128-LABEL: srem_v16i16:391; VBITS_GE_128: // %bb.0:392; VBITS_GE_128-NEXT: ldp q4, q1, [x1]393; VBITS_GE_128-NEXT: ptrue p0.s, vl4394; VBITS_GE_128-NEXT: ldr q0, [x0, #16]395; VBITS_GE_128-NEXT: sshll2 v2.4s, v1.8h, #0396; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0397; VBITS_GE_128-NEXT: sshll2 v5.4s, v4.8h, #0398; VBITS_GE_128-NEXT: sshll v16.4s, v0.4h, #0399; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s400; VBITS_GE_128-NEXT: ldr q3, [x0]401; VBITS_GE_128-NEXT: sshll2 v6.4s, v3.8h, #0402; VBITS_GE_128-NEXT: sshll v7.4s, v3.4h, #0403; VBITS_GE_128-NEXT: sdivr z5.s, p0/m, z5.s, z6.s404; VBITS_GE_128-NEXT: sshll v6.4s, v4.4h, #0405; VBITS_GE_128-NEXT: sdivr z6.s, p0/m, z6.s, z7.s406; VBITS_GE_128-NEXT: sshll v7.4s, v1.4h, #0407; VBITS_GE_128-NEXT: sdivr z7.s, p0/m, z7.s, z16.s408; VBITS_GE_128-NEXT: uzp1 v5.8h, v6.8h, v5.8h409; VBITS_GE_128-NEXT: mls v3.8h, v5.8h, v4.8h410; VBITS_GE_128-NEXT: uzp1 v2.8h, v7.8h, v2.8h411; VBITS_GE_128-NEXT: mls v0.8h, v2.8h, v1.8h412; VBITS_GE_128-NEXT: stp q3, q0, [x0]413; VBITS_GE_128-NEXT: ret414;415; VBITS_GE_256-LABEL: srem_v16i16:416; VBITS_GE_256: // %bb.0:417; VBITS_GE_256-NEXT: ptrue p0.h, vl16418; VBITS_GE_256-NEXT: ptrue p1.s, vl8419; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]420; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1]421; VBITS_GE_256-NEXT: sunpklo z2.s, z1.h422; VBITS_GE_256-NEXT: sunpklo z3.s, z0.h423; VBITS_GE_256-NEXT: movprfx z4, z0424; VBITS_GE_256-NEXT: ext z4.b, z4.b, z0.b, #16425; VBITS_GE_256-NEXT: sunpklo z4.s, z4.h426; VBITS_GE_256-NEXT: sdivr z2.s, p1/m, z2.s, z3.s427; VBITS_GE_256-NEXT: movprfx z3, z1428; VBITS_GE_256-NEXT: ext z3.b, z3.b, z1.b, #16429; VBITS_GE_256-NEXT: sunpklo z3.s, z3.h430; VBITS_GE_256-NEXT: sdivr z3.s, p1/m, z3.s, z4.s431; VBITS_GE_256-NEXT: ptrue p1.h, vl8432; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h433; VBITS_GE_256-NEXT: uzp1 z3.h, z3.h, z3.h434; VBITS_GE_256-NEXT: splice z2.h, p1, z2.h, z3.h435; VBITS_GE_256-NEXT: mls z0.h, p0/m, z2.h, z1.h436; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0]437; VBITS_GE_256-NEXT: ret438;439; VBITS_GE_512-LABEL: srem_v16i16:440; VBITS_GE_512: // %bb.0:441; VBITS_GE_512-NEXT: ptrue p0.h, vl16442; VBITS_GE_512-NEXT: ptrue p1.s, vl16443; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]444; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]445; VBITS_GE_512-NEXT: sunpklo z2.s, z1.h446; VBITS_GE_512-NEXT: sunpklo z3.s, z0.h447; VBITS_GE_512-NEXT: sdivr z2.s, p1/m, z2.s, z3.s448; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h449; VBITS_GE_512-NEXT: mls z0.h, p0/m, z2.h, z1.h450; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]451; VBITS_GE_512-NEXT: ret452 %op1 = load <16 x i16>, ptr %a453 %op2 = load <16 x i16>, ptr %b454 %res = srem <16 x i16> %op1, %op2455 store <16 x i16> %res, ptr %a456 ret void457}458 459define void @srem_v32i16(ptr %a, ptr %b) vscale_range(8,0) #0 {460; CHECK-LABEL: srem_v32i16:461; CHECK: // %bb.0:462; CHECK-NEXT: ptrue p0.h, vl32463; CHECK-NEXT: ptrue p1.s, vl32464; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]465; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]466; CHECK-NEXT: sunpklo z2.s, z1.h467; CHECK-NEXT: sunpklo z3.s, z0.h468; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s469; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h470; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h471; CHECK-NEXT: st1h { z0.h }, p0, [x0]472; CHECK-NEXT: ret473 %op1 = load <32 x i16>, ptr %a474 %op2 = load <32 x i16>, ptr %b475 %res = srem <32 x i16> %op1, %op2476 store <32 x i16> %res, ptr %a477 ret void478}479 480define void @srem_v64i16(ptr %a, ptr %b) vscale_range(16,0) #0 {481; CHECK-LABEL: srem_v64i16:482; CHECK: // %bb.0:483; CHECK-NEXT: ptrue p0.h, vl64484; CHECK-NEXT: ptrue p1.s, vl64485; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]486; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]487; CHECK-NEXT: sunpklo z2.s, z1.h488; CHECK-NEXT: sunpklo z3.s, z0.h489; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s490; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h491; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h492; CHECK-NEXT: st1h { z0.h }, p0, [x0]493; CHECK-NEXT: ret494 %op1 = load <64 x i16>, ptr %a495 %op2 = load <64 x i16>, ptr %b496 %res = srem <64 x i16> %op1, %op2497 store <64 x i16> %res, ptr %a498 ret void499}500 501define void @srem_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {502; CHECK-LABEL: srem_v128i16:503; CHECK: // %bb.0:504; CHECK-NEXT: ptrue p0.h, vl128505; CHECK-NEXT: ptrue p1.s, vl64506; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]507; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]508; CHECK-NEXT: sunpklo z2.s, z1.h509; CHECK-NEXT: sunpklo z3.s, z0.h510; CHECK-NEXT: movprfx z4, z0511; CHECK-NEXT: ext z4.b, z4.b, z0.b, #128512; CHECK-NEXT: sunpklo z4.s, z4.h513; CHECK-NEXT: sdivr z2.s, p1/m, z2.s, z3.s514; CHECK-NEXT: movprfx z3, z1515; CHECK-NEXT: ext z3.b, z3.b, z1.b, #128516; CHECK-NEXT: sunpklo z3.s, z3.h517; CHECK-NEXT: sdivr z3.s, p1/m, z3.s, z4.s518; CHECK-NEXT: ptrue p1.h, vl64519; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h520; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h521; CHECK-NEXT: splice z2.h, p1, z2.h, z3.h522; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h523; CHECK-NEXT: st1h { z0.h }, p0, [x0]524; CHECK-NEXT: ret525 %op1 = load <128 x i16>, ptr %a526 %op2 = load <128 x i16>, ptr %b527 %res = srem <128 x i16> %op1, %op2528 store <128 x i16> %res, ptr %a529 ret void530}531 532; Vector v2i32 sdiv are not legal for NEON so use SVE when available.533define <2 x i32> @srem_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(1,0) #0 {534; CHECK-LABEL: srem_v2i32:535; CHECK: // %bb.0:536; CHECK-NEXT: ptrue p0.s, vl2537; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1538; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0539; CHECK-NEXT: movprfx z2, z0540; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z1.s541; CHECK-NEXT: mls v0.2s, v2.2s, v1.2s542; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0543; CHECK-NEXT: ret544 %res = srem <2 x i32> %op1, %op2545 ret <2 x i32> %res546}547 548; Vector v4i32 sdiv are not legal for NEON so use SVE when available.549define <4 x i32> @srem_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(1,0) #0 {550; CHECK-LABEL: srem_v4i32:551; CHECK: // %bb.0:552; CHECK-NEXT: ptrue p0.s, vl4553; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1554; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0555; CHECK-NEXT: movprfx z2, z0556; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z1.s557; CHECK-NEXT: mls v0.4s, v2.4s, v1.4s558; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0559; CHECK-NEXT: ret560 %res = srem <4 x i32> %op1, %op2561 ret <4 x i32> %res562}563 564define void @srem_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {565; CHECK-LABEL: srem_v8i32:566; CHECK: // %bb.0:567; CHECK-NEXT: ptrue p0.s, vl8568; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]569; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]570; CHECK-NEXT: movprfx z2, z0571; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z1.s572; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s573; CHECK-NEXT: st1w { z0.s }, p0, [x0]574; CHECK-NEXT: ret575 %op1 = load <8 x i32>, ptr %a576 %op2 = load <8 x i32>, ptr %b577 %res = srem <8 x i32> %op1, %op2578 store <8 x i32> %res, ptr %a579 ret void580}581 582define void @srem_v16i32(ptr %a, ptr %b) #0 {583; VBITS_GE_128-LABEL: srem_v16i32:584; VBITS_GE_128: // %bb.0:585; VBITS_GE_128-NEXT: ldp q0, q3, [x1]586; VBITS_GE_128-NEXT: ptrue p0.s, vl4587; VBITS_GE_128-NEXT: ldp q1, q2, [x0]588; VBITS_GE_128-NEXT: ldp q16, q5, [x0, #32]589; VBITS_GE_128-NEXT: ldp q17, q6, [x1, #32]590; VBITS_GE_128-NEXT: movprfx z4, z1591; VBITS_GE_128-NEXT: sdiv z4.s, p0/m, z4.s, z0.s592; VBITS_GE_128-NEXT: movprfx z19, z2593; VBITS_GE_128-NEXT: sdiv z19.s, p0/m, z19.s, z3.s594; VBITS_GE_128-NEXT: movprfx z7, z5595; VBITS_GE_128-NEXT: sdiv z7.s, p0/m, z7.s, z6.s596; VBITS_GE_128-NEXT: movprfx z18, z16597; VBITS_GE_128-NEXT: sdiv z18.s, p0/m, z18.s, z17.s598; VBITS_GE_128-NEXT: mls v1.4s, v4.4s, v0.4s599; VBITS_GE_128-NEXT: mls v2.4s, v19.4s, v3.4s600; VBITS_GE_128-NEXT: mls v16.4s, v18.4s, v17.4s601; VBITS_GE_128-NEXT: mls v5.4s, v7.4s, v6.4s602; VBITS_GE_128-NEXT: stp q1, q2, [x0]603; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32]604; VBITS_GE_128-NEXT: ret605;606; VBITS_GE_256-LABEL: srem_v16i32:607; VBITS_GE_256: // %bb.0:608; VBITS_GE_256-NEXT: ptrue p0.s, vl8609; VBITS_GE_256-NEXT: mov x8, #8 // =0x8610; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]611; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]612; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x0]613; VBITS_GE_256-NEXT: ld1w { z4.s }, p0/z, [x1]614; VBITS_GE_256-NEXT: movprfx z2, z0615; VBITS_GE_256-NEXT: sdiv z2.s, p0/m, z2.s, z1.s616; VBITS_GE_256-NEXT: movprfx z5, z3617; VBITS_GE_256-NEXT: sdiv z5.s, p0/m, z5.s, z4.s618; VBITS_GE_256-NEXT: mls z0.s, p0/m, z2.s, z1.s619; VBITS_GE_256-NEXT: mls z3.s, p0/m, z5.s, z4.s620; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]621; VBITS_GE_256-NEXT: st1w { z3.s }, p0, [x0]622; VBITS_GE_256-NEXT: ret623;624; VBITS_GE_512-LABEL: srem_v16i32:625; VBITS_GE_512: // %bb.0:626; VBITS_GE_512-NEXT: ptrue p0.s, vl16627; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]628; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]629; VBITS_GE_512-NEXT: movprfx z2, z0630; VBITS_GE_512-NEXT: sdiv z2.s, p0/m, z2.s, z1.s631; VBITS_GE_512-NEXT: mls z0.s, p0/m, z2.s, z1.s632; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]633; VBITS_GE_512-NEXT: ret634 %op1 = load <16 x i32>, ptr %a635 %op2 = load <16 x i32>, ptr %b636 %res = srem <16 x i32> %op1, %op2637 store <16 x i32> %res, ptr %a638 ret void639}640 641define void @srem_v32i32(ptr %a, ptr %b) vscale_range(8,0) #0 {642; CHECK-LABEL: srem_v32i32:643; CHECK: // %bb.0:644; CHECK-NEXT: ptrue p0.s, vl32645; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]646; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]647; CHECK-NEXT: movprfx z2, z0648; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z1.s649; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s650; CHECK-NEXT: st1w { z0.s }, p0, [x0]651; CHECK-NEXT: ret652 %op1 = load <32 x i32>, ptr %a653 %op2 = load <32 x i32>, ptr %b654 %res = srem <32 x i32> %op1, %op2655 store <32 x i32> %res, ptr %a656 ret void657}658 659define void @srem_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {660; CHECK-LABEL: srem_v64i32:661; CHECK: // %bb.0:662; CHECK-NEXT: ptrue p0.s, vl64663; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]664; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]665; CHECK-NEXT: movprfx z2, z0666; CHECK-NEXT: sdiv z2.s, p0/m, z2.s, z1.s667; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s668; CHECK-NEXT: st1w { z0.s }, p0, [x0]669; CHECK-NEXT: ret670 %op1 = load <64 x i32>, ptr %a671 %op2 = load <64 x i32>, ptr %b672 %res = srem <64 x i32> %op1, %op2673 store <64 x i32> %res, ptr %a674 ret void675}676 677; Vector i64 sdiv are not legal for NEON so use SVE when available.678; FIXME: We should be able to improve the codegen for the 128 bits case here.679define <1 x i64> @srem_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(1,0) #0 {680; CHECK-LABEL: srem_v1i64:681; CHECK: // %bb.0:682; CHECK-NEXT: ptrue p0.d, vl1683; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1684; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0685; CHECK-NEXT: movprfx z2, z0686; CHECK-NEXT: sdiv z2.d, p0/m, z2.d, z1.d687; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d688; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0689; CHECK-NEXT: ret690 %res = srem <1 x i64> %op1, %op2691 ret <1 x i64> %res692}693 694; Vector i64 sdiv are not legal for NEON so use SVE when available.695; FIXME: We should be able to improve the codegen for the 128 bits case here.696define <2 x i64> @srem_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(1,0) #0 {697; CHECK-LABEL: srem_v2i64:698; CHECK: // %bb.0:699; CHECK-NEXT: ptrue p0.d, vl2700; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1701; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0702; CHECK-NEXT: movprfx z2, z0703; CHECK-NEXT: sdiv z2.d, p0/m, z2.d, z1.d704; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d705; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0706; CHECK-NEXT: ret707 %res = srem <2 x i64> %op1, %op2708 ret <2 x i64> %res709}710 711define void @srem_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {712; CHECK-LABEL: srem_v4i64:713; CHECK: // %bb.0:714; CHECK-NEXT: ptrue p0.d, vl4715; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]716; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]717; CHECK-NEXT: movprfx z2, z0718; CHECK-NEXT: sdiv z2.d, p0/m, z2.d, z1.d719; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d720; CHECK-NEXT: st1d { z0.d }, p0, [x0]721; CHECK-NEXT: ret722 %op1 = load <4 x i64>, ptr %a723 %op2 = load <4 x i64>, ptr %b724 %res = srem <4 x i64> %op1, %op2725 store <4 x i64> %res, ptr %a726 ret void727}728 729define void @srem_v8i64(ptr %a, ptr %b) #0 {730; VBITS_GE_128-LABEL: srem_v8i64:731; VBITS_GE_128: // %bb.0:732; VBITS_GE_128-NEXT: ldp q0, q3, [x1]733; VBITS_GE_128-NEXT: ptrue p0.d, vl2734; VBITS_GE_128-NEXT: ldp q1, q2, [x0]735; VBITS_GE_128-NEXT: ldp q16, q5, [x0, #32]736; VBITS_GE_128-NEXT: ldp q17, q6, [x1, #32]737; VBITS_GE_128-NEXT: movprfx z4, z1738; VBITS_GE_128-NEXT: sdiv z4.d, p0/m, z4.d, z0.d739; VBITS_GE_128-NEXT: movprfx z19, z2740; VBITS_GE_128-NEXT: sdiv z19.d, p0/m, z19.d, z3.d741; VBITS_GE_128-NEXT: movprfx z7, z5742; VBITS_GE_128-NEXT: sdiv z7.d, p0/m, z7.d, z6.d743; VBITS_GE_128-NEXT: movprfx z18, z16744; VBITS_GE_128-NEXT: sdiv z18.d, p0/m, z18.d, z17.d745; VBITS_GE_128-NEXT: msb z0.d, p0/m, z4.d, z1.d746; VBITS_GE_128-NEXT: mls z2.d, p0/m, z19.d, z3.d747; VBITS_GE_128-NEXT: mls z16.d, p0/m, z18.d, z17.d748; VBITS_GE_128-NEXT: mls z5.d, p0/m, z7.d, z6.d749; VBITS_GE_128-NEXT: stp q0, q2, [x0]750; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32]751; VBITS_GE_128-NEXT: ret752;753; VBITS_GE_256-LABEL: srem_v8i64:754; VBITS_GE_256: // %bb.0:755; VBITS_GE_256-NEXT: ptrue p0.d, vl4756; VBITS_GE_256-NEXT: mov x8, #4 // =0x4757; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]758; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]759; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x0]760; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1]761; VBITS_GE_256-NEXT: movprfx z2, z0762; VBITS_GE_256-NEXT: sdiv z2.d, p0/m, z2.d, z1.d763; VBITS_GE_256-NEXT: movprfx z5, z3764; VBITS_GE_256-NEXT: sdiv z5.d, p0/m, z5.d, z4.d765; VBITS_GE_256-NEXT: mls z0.d, p0/m, z2.d, z1.d766; VBITS_GE_256-NEXT: mls z3.d, p0/m, z5.d, z4.d767; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]768; VBITS_GE_256-NEXT: st1d { z3.d }, p0, [x0]769; VBITS_GE_256-NEXT: ret770;771; VBITS_GE_512-LABEL: srem_v8i64:772; VBITS_GE_512: // %bb.0:773; VBITS_GE_512-NEXT: ptrue p0.d, vl8774; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]775; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]776; VBITS_GE_512-NEXT: movprfx z2, z0777; VBITS_GE_512-NEXT: sdiv z2.d, p0/m, z2.d, z1.d778; VBITS_GE_512-NEXT: mls z0.d, p0/m, z2.d, z1.d779; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]780; VBITS_GE_512-NEXT: ret781 %op1 = load <8 x i64>, ptr %a782 %op2 = load <8 x i64>, ptr %b783 %res = srem <8 x i64> %op1, %op2784 store <8 x i64> %res, ptr %a785 ret void786}787 788define void @srem_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 {789; CHECK-LABEL: srem_v16i64:790; CHECK: // %bb.0:791; CHECK-NEXT: ptrue p0.d, vl16792; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]793; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]794; CHECK-NEXT: movprfx z2, z0795; CHECK-NEXT: sdiv z2.d, p0/m, z2.d, z1.d796; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d797; CHECK-NEXT: st1d { z0.d }, p0, [x0]798; CHECK-NEXT: ret799 %op1 = load <16 x i64>, ptr %a800 %op2 = load <16 x i64>, ptr %b801 %res = srem <16 x i64> %op1, %op2802 store <16 x i64> %res, ptr %a803 ret void804}805 806define void @srem_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {807; CHECK-LABEL: srem_v32i64:808; CHECK: // %bb.0:809; CHECK-NEXT: ptrue p0.d, vl32810; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]811; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]812; CHECK-NEXT: movprfx z2, z0813; CHECK-NEXT: sdiv z2.d, p0/m, z2.d, z1.d814; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d815; CHECK-NEXT: st1d { z0.d }, p0, [x0]816; CHECK-NEXT: ret817 %op1 = load <32 x i64>, ptr %a818 %op2 = load <32 x i64>, ptr %b819 %res = srem <32 x i64> %op1, %op2820 store <32 x i64> %res, ptr %a821 ret void822}823 824;825; UREM826;827 828; Vector vXi8 udiv are not legal for NEON so use SVE when available.829; FIXME: We should be able to improve the codegen for >= 256 bits here.830define <8 x i8> @urem_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 {831; VBITS_GE_128-LABEL: urem_v8i8:832; VBITS_GE_128: // %bb.0:833; VBITS_GE_128-NEXT: ushll v2.8h, v1.8b, #0834; VBITS_GE_128-NEXT: ushll v3.8h, v0.8b, #0835; VBITS_GE_128-NEXT: ptrue p0.s, vl4836; VBITS_GE_128-NEXT: ushll2 v4.4s, v2.8h, #0837; VBITS_GE_128-NEXT: ushll2 v5.4s, v3.8h, #0838; VBITS_GE_128-NEXT: ushll v2.4s, v2.4h, #0839; VBITS_GE_128-NEXT: ushll v3.4s, v3.4h, #0840; VBITS_GE_128-NEXT: udivr z4.s, p0/m, z4.s, z5.s841; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s842; VBITS_GE_128-NEXT: uzp1 v2.8h, v2.8h, v4.8h843; VBITS_GE_128-NEXT: xtn v2.8b, v2.8h844; VBITS_GE_128-NEXT: mls v0.8b, v2.8b, v1.8b845; VBITS_GE_128-NEXT: ret846;847; VBITS_GE_256-LABEL: urem_v8i8:848; VBITS_GE_256: // %bb.0:849; VBITS_GE_256-NEXT: // kill: def $d1 killed $d1 def $z1850; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 def $z0851; VBITS_GE_256-NEXT: uunpklo z2.h, z1.b852; VBITS_GE_256-NEXT: uunpklo z3.h, z0.b853; VBITS_GE_256-NEXT: ptrue p0.s, vl8854; VBITS_GE_256-NEXT: uunpklo z2.s, z2.h855; VBITS_GE_256-NEXT: uunpklo z3.s, z3.h856; VBITS_GE_256-NEXT: udivr z2.s, p0/m, z2.s, z3.s857; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h858; VBITS_GE_256-NEXT: umov w8, v2.h[0]859; VBITS_GE_256-NEXT: umov w9, v2.h[1]860; VBITS_GE_256-NEXT: fmov s3, w8861; VBITS_GE_256-NEXT: umov w8, v2.h[2]862; VBITS_GE_256-NEXT: mov v3.b[1], w9863; VBITS_GE_256-NEXT: mov v3.b[2], w8864; VBITS_GE_256-NEXT: umov w8, v2.h[3]865; VBITS_GE_256-NEXT: mov v3.b[3], w8866; VBITS_GE_256-NEXT: umov w8, v2.h[4]867; VBITS_GE_256-NEXT: mov v3.b[4], w8868; VBITS_GE_256-NEXT: umov w8, v2.h[5]869; VBITS_GE_256-NEXT: mov v3.b[5], w8870; VBITS_GE_256-NEXT: umov w8, v2.h[6]871; VBITS_GE_256-NEXT: mov v3.b[6], w8872; VBITS_GE_256-NEXT: umov w8, v2.h[7]873; VBITS_GE_256-NEXT: mov v3.b[7], w8874; VBITS_GE_256-NEXT: mls v0.8b, v3.8b, v1.8b875; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 killed $z0876; VBITS_GE_256-NEXT: ret877;878; VBITS_GE_512-LABEL: urem_v8i8:879; VBITS_GE_512: // %bb.0:880; VBITS_GE_512-NEXT: // kill: def $d1 killed $d1 def $z1881; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 def $z0882; VBITS_GE_512-NEXT: uunpklo z2.h, z1.b883; VBITS_GE_512-NEXT: uunpklo z3.h, z0.b884; VBITS_GE_512-NEXT: ptrue p0.s, vl8885; VBITS_GE_512-NEXT: uunpklo z2.s, z2.h886; VBITS_GE_512-NEXT: uunpklo z3.s, z3.h887; VBITS_GE_512-NEXT: udivr z2.s, p0/m, z2.s, z3.s888; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h889; VBITS_GE_512-NEXT: umov w8, v2.h[0]890; VBITS_GE_512-NEXT: umov w9, v2.h[1]891; VBITS_GE_512-NEXT: fmov s3, w8892; VBITS_GE_512-NEXT: umov w8, v2.h[2]893; VBITS_GE_512-NEXT: mov v3.b[1], w9894; VBITS_GE_512-NEXT: mov v3.b[2], w8895; VBITS_GE_512-NEXT: umov w8, v2.h[3]896; VBITS_GE_512-NEXT: mov v3.b[3], w8897; VBITS_GE_512-NEXT: umov w8, v2.h[4]898; VBITS_GE_512-NEXT: mov v3.b[4], w8899; VBITS_GE_512-NEXT: umov w8, v2.h[5]900; VBITS_GE_512-NEXT: mov v3.b[5], w8901; VBITS_GE_512-NEXT: umov w8, v2.h[6]902; VBITS_GE_512-NEXT: mov v3.b[6], w8903; VBITS_GE_512-NEXT: umov w8, v2.h[7]904; VBITS_GE_512-NEXT: mov v3.b[7], w8905; VBITS_GE_512-NEXT: mls v0.8b, v3.8b, v1.8b906; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 killed $z0907; VBITS_GE_512-NEXT: ret908 %res = urem <8 x i8> %op1, %op2909 ret <8 x i8> %res910}911 912define <16 x i8> @urem_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 {913; VBITS_GE_128-LABEL: urem_v16i8:914; VBITS_GE_128: // %bb.0:915; VBITS_GE_128-NEXT: ushll2 v2.8h, v1.16b, #0916; VBITS_GE_128-NEXT: ushll2 v3.8h, v0.16b, #0917; VBITS_GE_128-NEXT: ptrue p0.s, vl4918; VBITS_GE_128-NEXT: ushll2 v4.4s, v2.8h, #0919; VBITS_GE_128-NEXT: ushll2 v5.4s, v3.8h, #0920; VBITS_GE_128-NEXT: ushll v2.4s, v2.4h, #0921; VBITS_GE_128-NEXT: ushll v3.4s, v3.4h, #0922; VBITS_GE_128-NEXT: udivr z4.s, p0/m, z4.s, z5.s923; VBITS_GE_128-NEXT: ushll v5.8h, v0.8b, #0924; VBITS_GE_128-NEXT: ushll2 v7.4s, v5.8h, #0925; VBITS_GE_128-NEXT: ushll v5.4s, v5.4h, #0926; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s927; VBITS_GE_128-NEXT: ushll v3.8h, v1.8b, #0928; VBITS_GE_128-NEXT: ushll2 v6.4s, v3.8h, #0929; VBITS_GE_128-NEXT: ushll v3.4s, v3.4h, #0930; VBITS_GE_128-NEXT: udivr z6.s, p0/m, z6.s, z7.s931; VBITS_GE_128-NEXT: uzp1 v2.8h, v2.8h, v4.8h932; VBITS_GE_128-NEXT: udivr z3.s, p0/m, z3.s, z5.s933; VBITS_GE_128-NEXT: uzp1 v3.8h, v3.8h, v6.8h934; VBITS_GE_128-NEXT: uzp1 v2.16b, v3.16b, v2.16b935; VBITS_GE_128-NEXT: mls v0.16b, v2.16b, v1.16b936; VBITS_GE_128-NEXT: ret937;938; VBITS_GE_256-LABEL: urem_v16i8:939; VBITS_GE_256: // %bb.0:940; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1941; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0942; VBITS_GE_256-NEXT: uunpklo z2.h, z1.b943; VBITS_GE_256-NEXT: uunpklo z3.h, z0.b944; VBITS_GE_256-NEXT: ptrue p0.s, vl8945; VBITS_GE_256-NEXT: uunpklo z4.s, z2.h946; VBITS_GE_256-NEXT: uunpklo z5.s, z3.h947; VBITS_GE_256-NEXT: ext z2.b, z2.b, z2.b, #16948; VBITS_GE_256-NEXT: ext z3.b, z3.b, z3.b, #16949; VBITS_GE_256-NEXT: uunpklo z2.s, z2.h950; VBITS_GE_256-NEXT: uunpklo z3.s, z3.h951; VBITS_GE_256-NEXT: udivr z4.s, p0/m, z4.s, z5.s952; VBITS_GE_256-NEXT: udivr z2.s, p0/m, z2.s, z3.s953; VBITS_GE_256-NEXT: ptrue p0.h, vl8954; VBITS_GE_256-NEXT: uzp1 z3.h, z4.h, z4.h955; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h956; VBITS_GE_256-NEXT: splice z3.h, p0, z3.h, z2.h957; VBITS_GE_256-NEXT: uzp1 z2.b, z3.b, z3.b958; VBITS_GE_256-NEXT: mls v0.16b, v2.16b, v1.16b959; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z0960; VBITS_GE_256-NEXT: ret961;962; VBITS_GE_512-LABEL: urem_v16i8:963; VBITS_GE_512: // %bb.0:964; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1965; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0966; VBITS_GE_512-NEXT: uunpklo z2.h, z1.b967; VBITS_GE_512-NEXT: uunpklo z3.h, z0.b968; VBITS_GE_512-NEXT: ptrue p0.s, vl16969; VBITS_GE_512-NEXT: uunpklo z2.s, z2.h970; VBITS_GE_512-NEXT: uunpklo z3.s, z3.h971; VBITS_GE_512-NEXT: udivr z2.s, p0/m, z2.s, z3.s972; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h973; VBITS_GE_512-NEXT: uzp1 z2.b, z2.b, z2.b974; VBITS_GE_512-NEXT: mls v0.16b, v2.16b, v1.16b975; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z0976; VBITS_GE_512-NEXT: ret977 %res = urem <16 x i8> %op1, %op2978 ret <16 x i8> %res979}980 981define void @urem_v32i8(ptr %a, ptr %b) vscale_range(8,0) #0 {982; CHECK-LABEL: urem_v32i8:983; CHECK: // %bb.0:984; CHECK-NEXT: ptrue p0.b, vl32985; CHECK-NEXT: ptrue p1.s, vl32986; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]987; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]988; CHECK-NEXT: uunpklo z2.h, z1.b989; CHECK-NEXT: uunpklo z3.h, z0.b990; CHECK-NEXT: uunpklo z2.s, z2.h991; CHECK-NEXT: uunpklo z3.s, z3.h992; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s993; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h994; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b995; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b996; CHECK-NEXT: st1b { z0.b }, p0, [x0]997; CHECK-NEXT: ret998 %op1 = load <32 x i8>, ptr %a999 %op2 = load <32 x i8>, ptr %b1000 %res = urem <32 x i8> %op1, %op21001 store <32 x i8> %res, ptr %a1002 ret void1003}1004 1005define void @urem_v64i8(ptr %a, ptr %b) vscale_range(16,0) #0 {1006; CHECK-LABEL: urem_v64i8:1007; CHECK: // %bb.0:1008; CHECK-NEXT: ptrue p0.b, vl641009; CHECK-NEXT: ptrue p1.s, vl641010; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]1011; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]1012; CHECK-NEXT: uunpklo z2.h, z1.b1013; CHECK-NEXT: uunpklo z3.h, z0.b1014; CHECK-NEXT: uunpklo z2.s, z2.h1015; CHECK-NEXT: uunpklo z3.s, z3.h1016; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s1017; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h1018; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b1019; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b1020; CHECK-NEXT: st1b { z0.b }, p0, [x0]1021; CHECK-NEXT: ret1022 %op1 = load <64 x i8>, ptr %a1023 %op2 = load <64 x i8>, ptr %b1024 %res = urem <64 x i8> %op1, %op21025 store <64 x i8> %res, ptr %a1026 ret void1027}1028 1029define void @urem_v128i8(ptr %a, ptr %b) vscale_range(16,0) #0 {1030; CHECK-LABEL: urem_v128i8:1031; CHECK: // %bb.0:1032; CHECK-NEXT: ptrue p0.b, vl1281033; CHECK-NEXT: ptrue p1.s, vl641034; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]1035; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]1036; CHECK-NEXT: uunpklo z2.h, z1.b1037; CHECK-NEXT: uunpklo z3.h, z0.b1038; CHECK-NEXT: uunpklo z4.s, z2.h1039; CHECK-NEXT: uunpklo z5.s, z3.h1040; CHECK-NEXT: ext z2.b, z2.b, z2.b, #1281041; CHECK-NEXT: ext z3.b, z3.b, z3.b, #1281042; CHECK-NEXT: uunpklo z2.s, z2.h1043; CHECK-NEXT: uunpklo z3.s, z3.h1044; CHECK-NEXT: udivr z4.s, p1/m, z4.s, z5.s1045; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s1046; CHECK-NEXT: ptrue p1.h, vl641047; CHECK-NEXT: uzp1 z3.h, z4.h, z4.h1048; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h1049; CHECK-NEXT: splice z3.h, p1, z3.h, z2.h1050; CHECK-NEXT: uzp1 z2.b, z3.b, z3.b1051; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b1052; CHECK-NEXT: st1b { z0.b }, p0, [x0]1053; CHECK-NEXT: ret1054 %op1 = load <128 x i8>, ptr %a1055 %op2 = load <128 x i8>, ptr %b1056 %res = urem <128 x i8> %op1, %op21057 store <128 x i8> %res, ptr %a1058 ret void1059}1060 1061define void @urem_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {1062; CHECK-LABEL: urem_v256i8:1063; CHECK: // %bb.0:1064; CHECK-NEXT: ptrue p0.b, vl2561065; CHECK-NEXT: ptrue p1.s, vl641066; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]1067; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]1068; CHECK-NEXT: uunpklo z2.h, z1.b1069; CHECK-NEXT: uunpklo z3.h, z0.b1070; CHECK-NEXT: uunpklo z4.s, z2.h1071; CHECK-NEXT: uunpklo z5.s, z3.h1072; CHECK-NEXT: ext z2.b, z2.b, z2.b, #1281073; CHECK-NEXT: ext z3.b, z3.b, z3.b, #1281074; CHECK-NEXT: uunpklo z2.s, z2.h1075; CHECK-NEXT: uunpklo z3.s, z3.h1076; CHECK-NEXT: udivr z4.s, p1/m, z4.s, z5.s1077; CHECK-NEXT: movprfx z5, z01078; CHECK-NEXT: ext z5.b, z5.b, z0.b, #1281079; CHECK-NEXT: uunpklo z5.h, z5.b1080; CHECK-NEXT: uunpklo z7.s, z5.h1081; CHECK-NEXT: ext z5.b, z5.b, z5.b, #1281082; CHECK-NEXT: uunpklo z5.s, z5.h1083; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s1084; CHECK-NEXT: movprfx z3, z11085; CHECK-NEXT: ext z3.b, z3.b, z1.b, #1281086; CHECK-NEXT: uunpklo z3.h, z3.b1087; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h1088; CHECK-NEXT: uunpklo z6.s, z3.h1089; CHECK-NEXT: ext z3.b, z3.b, z3.b, #1281090; CHECK-NEXT: uunpklo z3.s, z3.h1091; CHECK-NEXT: udivr z6.s, p1/m, z6.s, z7.s1092; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h1093; CHECK-NEXT: udivr z3.s, p1/m, z3.s, z5.s1094; CHECK-NEXT: ptrue p1.h, vl641095; CHECK-NEXT: splice z4.h, p1, z4.h, z2.h1096; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h1097; CHECK-NEXT: uzp1 z2.b, z4.b, z4.b1098; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h1099; CHECK-NEXT: splice z5.h, p1, z5.h, z3.h1100; CHECK-NEXT: ptrue p1.b, vl1281101; CHECK-NEXT: uzp1 z3.b, z5.b, z5.b1102; CHECK-NEXT: splice z2.b, p1, z2.b, z3.b1103; CHECK-NEXT: mls z0.b, p0/m, z2.b, z1.b1104; CHECK-NEXT: st1b { z0.b }, p0, [x0]1105; CHECK-NEXT: ret1106 %op1 = load <256 x i8>, ptr %a1107 %op2 = load <256 x i8>, ptr %b1108 %res = urem <256 x i8> %op1, %op21109 store <256 x i8> %res, ptr %a1110 ret void1111}1112 1113; Vector vXi16 udiv are not legal for NEON so use SVE when available.1114; FIXME: We should be able to improve the codegen for >= 256 bits here.1115define <4 x i16> @urem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 {1116; VBITS_GE_128-LABEL: urem_v4i16:1117; VBITS_GE_128: // %bb.0:1118; VBITS_GE_128-NEXT: ushll v2.4s, v1.4h, #01119; VBITS_GE_128-NEXT: ushll v3.4s, v0.4h, #01120; VBITS_GE_128-NEXT: ptrue p0.s, vl41121; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s1122; VBITS_GE_128-NEXT: xtn v2.4h, v2.4s1123; VBITS_GE_128-NEXT: mls v0.4h, v2.4h, v1.4h1124; VBITS_GE_128-NEXT: ret1125;1126; VBITS_GE_256-LABEL: urem_v4i16:1127; VBITS_GE_256: // %bb.0:1128; VBITS_GE_256-NEXT: ushll v2.4s, v1.4h, #01129; VBITS_GE_256-NEXT: ushll v3.4s, v0.4h, #01130; VBITS_GE_256-NEXT: ptrue p0.s, vl41131; VBITS_GE_256-NEXT: udivr z2.s, p0/m, z2.s, z3.s1132; VBITS_GE_256-NEXT: mov w8, v2.s[1]1133; VBITS_GE_256-NEXT: mov v3.16b, v2.16b1134; VBITS_GE_256-NEXT: mov w9, v2.s[2]1135; VBITS_GE_256-NEXT: mov v3.h[1], w81136; VBITS_GE_256-NEXT: mov w8, v2.s[3]1137; VBITS_GE_256-NEXT: mov v3.h[2], w91138; VBITS_GE_256-NEXT: mov v3.h[3], w81139; VBITS_GE_256-NEXT: mls v0.4h, v3.4h, v1.4h1140; VBITS_GE_256-NEXT: ret1141;1142; VBITS_GE_512-LABEL: urem_v4i16:1143; VBITS_GE_512: // %bb.0:1144; VBITS_GE_512-NEXT: ushll v2.4s, v1.4h, #01145; VBITS_GE_512-NEXT: ushll v3.4s, v0.4h, #01146; VBITS_GE_512-NEXT: ptrue p0.s, vl41147; VBITS_GE_512-NEXT: udivr z2.s, p0/m, z2.s, z3.s1148; VBITS_GE_512-NEXT: mov w8, v2.s[1]1149; VBITS_GE_512-NEXT: mov v3.16b, v2.16b1150; VBITS_GE_512-NEXT: mov w9, v2.s[2]1151; VBITS_GE_512-NEXT: mov v3.h[1], w81152; VBITS_GE_512-NEXT: mov w8, v2.s[3]1153; VBITS_GE_512-NEXT: mov v3.h[2], w91154; VBITS_GE_512-NEXT: mov v3.h[3], w81155; VBITS_GE_512-NEXT: mls v0.4h, v3.4h, v1.4h1156; VBITS_GE_512-NEXT: ret1157 %res = urem <4 x i16> %op1, %op21158 ret <4 x i16> %res1159}1160 1161define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 {1162; VBITS_GE_128-LABEL: urem_v8i16:1163; VBITS_GE_128: // %bb.0:1164; VBITS_GE_128-NEXT: ushll2 v2.4s, v1.8h, #01165; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #01166; VBITS_GE_128-NEXT: ptrue p0.s, vl41167; VBITS_GE_128-NEXT: ushll v4.4s, v0.4h, #01168; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s1169; VBITS_GE_128-NEXT: ushll v3.4s, v1.4h, #01170; VBITS_GE_128-NEXT: udivr z3.s, p0/m, z3.s, z4.s1171; VBITS_GE_128-NEXT: uzp1 v2.8h, v3.8h, v2.8h1172; VBITS_GE_128-NEXT: mls v0.8h, v2.8h, v1.8h1173; VBITS_GE_128-NEXT: ret1174;1175; VBITS_GE_256-LABEL: urem_v8i16:1176; VBITS_GE_256: // %bb.0:1177; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z11178; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z01179; VBITS_GE_256-NEXT: uunpklo z2.s, z1.h1180; VBITS_GE_256-NEXT: uunpklo z3.s, z0.h1181; VBITS_GE_256-NEXT: ptrue p0.s, vl81182; VBITS_GE_256-NEXT: udivr z2.s, p0/m, z2.s, z3.s1183; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h1184; VBITS_GE_256-NEXT: mls v0.8h, v2.8h, v1.8h1185; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 killed $z01186; VBITS_GE_256-NEXT: ret1187;1188; VBITS_GE_512-LABEL: urem_v8i16:1189; VBITS_GE_512: // %bb.0:1190; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z11191; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z01192; VBITS_GE_512-NEXT: uunpklo z2.s, z1.h1193; VBITS_GE_512-NEXT: uunpklo z3.s, z0.h1194; VBITS_GE_512-NEXT: ptrue p0.s, vl81195; VBITS_GE_512-NEXT: udivr z2.s, p0/m, z2.s, z3.s1196; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h1197; VBITS_GE_512-NEXT: mls v0.8h, v2.8h, v1.8h1198; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 killed $z01199; VBITS_GE_512-NEXT: ret1200 %res = urem <8 x i16> %op1, %op21201 ret <8 x i16> %res1202}1203 1204define void @urem_v16i16(ptr %a, ptr %b) #0 {1205; VBITS_GE_128-LABEL: urem_v16i16:1206; VBITS_GE_128: // %bb.0:1207; VBITS_GE_128-NEXT: ldp q4, q1, [x1]1208; VBITS_GE_128-NEXT: ptrue p0.s, vl41209; VBITS_GE_128-NEXT: ldr q0, [x0, #16]1210; VBITS_GE_128-NEXT: ushll2 v2.4s, v1.8h, #01211; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #01212; VBITS_GE_128-NEXT: ushll2 v5.4s, v4.8h, #01213; VBITS_GE_128-NEXT: ushll v16.4s, v0.4h, #01214; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s1215; VBITS_GE_128-NEXT: ldr q3, [x0]1216; VBITS_GE_128-NEXT: ushll2 v6.4s, v3.8h, #01217; VBITS_GE_128-NEXT: ushll v7.4s, v3.4h, #01218; VBITS_GE_128-NEXT: udivr z5.s, p0/m, z5.s, z6.s1219; VBITS_GE_128-NEXT: ushll v6.4s, v4.4h, #01220; VBITS_GE_128-NEXT: udivr z6.s, p0/m, z6.s, z7.s1221; VBITS_GE_128-NEXT: ushll v7.4s, v1.4h, #01222; VBITS_GE_128-NEXT: udivr z7.s, p0/m, z7.s, z16.s1223; VBITS_GE_128-NEXT: uzp1 v5.8h, v6.8h, v5.8h1224; VBITS_GE_128-NEXT: mls v3.8h, v5.8h, v4.8h1225; VBITS_GE_128-NEXT: uzp1 v2.8h, v7.8h, v2.8h1226; VBITS_GE_128-NEXT: mls v0.8h, v2.8h, v1.8h1227; VBITS_GE_128-NEXT: stp q3, q0, [x0]1228; VBITS_GE_128-NEXT: ret1229;1230; VBITS_GE_256-LABEL: urem_v16i16:1231; VBITS_GE_256: // %bb.0:1232; VBITS_GE_256-NEXT: ptrue p0.h, vl161233; VBITS_GE_256-NEXT: ptrue p1.s, vl81234; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]1235; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1]1236; VBITS_GE_256-NEXT: uunpklo z2.s, z1.h1237; VBITS_GE_256-NEXT: uunpklo z3.s, z0.h1238; VBITS_GE_256-NEXT: movprfx z4, z01239; VBITS_GE_256-NEXT: ext z4.b, z4.b, z0.b, #161240; VBITS_GE_256-NEXT: uunpklo z4.s, z4.h1241; VBITS_GE_256-NEXT: udivr z2.s, p1/m, z2.s, z3.s1242; VBITS_GE_256-NEXT: movprfx z3, z11243; VBITS_GE_256-NEXT: ext z3.b, z3.b, z1.b, #161244; VBITS_GE_256-NEXT: uunpklo z3.s, z3.h1245; VBITS_GE_256-NEXT: udivr z3.s, p1/m, z3.s, z4.s1246; VBITS_GE_256-NEXT: ptrue p1.h, vl81247; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h1248; VBITS_GE_256-NEXT: uzp1 z3.h, z3.h, z3.h1249; VBITS_GE_256-NEXT: splice z2.h, p1, z2.h, z3.h1250; VBITS_GE_256-NEXT: mls z0.h, p0/m, z2.h, z1.h1251; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0]1252; VBITS_GE_256-NEXT: ret1253;1254; VBITS_GE_512-LABEL: urem_v16i16:1255; VBITS_GE_512: // %bb.0:1256; VBITS_GE_512-NEXT: ptrue p0.h, vl161257; VBITS_GE_512-NEXT: ptrue p1.s, vl161258; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]1259; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]1260; VBITS_GE_512-NEXT: uunpklo z2.s, z1.h1261; VBITS_GE_512-NEXT: uunpklo z3.s, z0.h1262; VBITS_GE_512-NEXT: udivr z2.s, p1/m, z2.s, z3.s1263; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h1264; VBITS_GE_512-NEXT: mls z0.h, p0/m, z2.h, z1.h1265; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]1266; VBITS_GE_512-NEXT: ret1267 %op1 = load <16 x i16>, ptr %a1268 %op2 = load <16 x i16>, ptr %b1269 %res = urem <16 x i16> %op1, %op21270 store <16 x i16> %res, ptr %a1271 ret void1272}1273 1274define void @urem_v32i16(ptr %a, ptr %b) vscale_range(8,0) #0 {1275; CHECK-LABEL: urem_v32i16:1276; CHECK: // %bb.0:1277; CHECK-NEXT: ptrue p0.h, vl321278; CHECK-NEXT: ptrue p1.s, vl321279; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1280; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]1281; CHECK-NEXT: uunpklo z2.s, z1.h1282; CHECK-NEXT: uunpklo z3.s, z0.h1283; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s1284; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h1285; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h1286; CHECK-NEXT: st1h { z0.h }, p0, [x0]1287; CHECK-NEXT: ret1288 %op1 = load <32 x i16>, ptr %a1289 %op2 = load <32 x i16>, ptr %b1290 %res = urem <32 x i16> %op1, %op21291 store <32 x i16> %res, ptr %a1292 ret void1293}1294 1295define void @urem_v64i16(ptr %a, ptr %b) vscale_range(16,0) #0 {1296; CHECK-LABEL: urem_v64i16:1297; CHECK: // %bb.0:1298; CHECK-NEXT: ptrue p0.h, vl641299; CHECK-NEXT: ptrue p1.s, vl641300; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1301; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]1302; CHECK-NEXT: uunpklo z2.s, z1.h1303; CHECK-NEXT: uunpklo z3.s, z0.h1304; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s1305; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h1306; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h1307; CHECK-NEXT: st1h { z0.h }, p0, [x0]1308; CHECK-NEXT: ret1309 %op1 = load <64 x i16>, ptr %a1310 %op2 = load <64 x i16>, ptr %b1311 %res = urem <64 x i16> %op1, %op21312 store <64 x i16> %res, ptr %a1313 ret void1314}1315 1316define void @urem_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {1317; CHECK-LABEL: urem_v128i16:1318; CHECK: // %bb.0:1319; CHECK-NEXT: ptrue p0.h, vl1281320; CHECK-NEXT: ptrue p1.s, vl641321; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1322; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]1323; CHECK-NEXT: uunpklo z2.s, z1.h1324; CHECK-NEXT: uunpklo z3.s, z0.h1325; CHECK-NEXT: movprfx z4, z01326; CHECK-NEXT: ext z4.b, z4.b, z0.b, #1281327; CHECK-NEXT: uunpklo z4.s, z4.h1328; CHECK-NEXT: udivr z2.s, p1/m, z2.s, z3.s1329; CHECK-NEXT: movprfx z3, z11330; CHECK-NEXT: ext z3.b, z3.b, z1.b, #1281331; CHECK-NEXT: uunpklo z3.s, z3.h1332; CHECK-NEXT: udivr z3.s, p1/m, z3.s, z4.s1333; CHECK-NEXT: ptrue p1.h, vl641334; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h1335; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h1336; CHECK-NEXT: splice z2.h, p1, z2.h, z3.h1337; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h1338; CHECK-NEXT: st1h { z0.h }, p0, [x0]1339; CHECK-NEXT: ret1340 %op1 = load <128 x i16>, ptr %a1341 %op2 = load <128 x i16>, ptr %b1342 %res = urem <128 x i16> %op1, %op21343 store <128 x i16> %res, ptr %a1344 ret void1345}1346 1347; Vector v2i32 udiv are not legal for NEON so use SVE when available.1348define <2 x i32> @urem_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(1,0) #0 {1349; CHECK-LABEL: urem_v2i32:1350; CHECK: // %bb.0:1351; CHECK-NEXT: ptrue p0.s, vl21352; CHECK-NEXT: // kill: def $d1 killed $d1 def $z11353; CHECK-NEXT: // kill: def $d0 killed $d0 def $z01354; CHECK-NEXT: movprfx z2, z01355; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z1.s1356; CHECK-NEXT: mls v0.2s, v2.2s, v1.2s1357; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z01358; CHECK-NEXT: ret1359 %res = urem <2 x i32> %op1, %op21360 ret <2 x i32> %res1361}1362 1363; Vector v4i32 udiv are not legal for NEON so use SVE when available.1364define <4 x i32> @urem_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(1,0) #0 {1365; CHECK-LABEL: urem_v4i32:1366; CHECK: // %bb.0:1367; CHECK-NEXT: ptrue p0.s, vl41368; CHECK-NEXT: // kill: def $q1 killed $q1 def $z11369; CHECK-NEXT: // kill: def $q0 killed $q0 def $z01370; CHECK-NEXT: movprfx z2, z01371; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z1.s1372; CHECK-NEXT: mls v0.4s, v2.4s, v1.4s1373; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z01374; CHECK-NEXT: ret1375 %res = urem <4 x i32> %op1, %op21376 ret <4 x i32> %res1377}1378 1379define void @urem_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {1380; CHECK-LABEL: urem_v8i32:1381; CHECK: // %bb.0:1382; CHECK-NEXT: ptrue p0.s, vl81383; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1384; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]1385; CHECK-NEXT: movprfx z2, z01386; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z1.s1387; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s1388; CHECK-NEXT: st1w { z0.s }, p0, [x0]1389; CHECK-NEXT: ret1390 %op1 = load <8 x i32>, ptr %a1391 %op2 = load <8 x i32>, ptr %b1392 %res = urem <8 x i32> %op1, %op21393 store <8 x i32> %res, ptr %a1394 ret void1395}1396 1397define void @urem_v16i32(ptr %a, ptr %b) #0 {1398; VBITS_GE_128-LABEL: urem_v16i32:1399; VBITS_GE_128: // %bb.0:1400; VBITS_GE_128-NEXT: ldp q0, q3, [x1]1401; VBITS_GE_128-NEXT: ptrue p0.s, vl41402; VBITS_GE_128-NEXT: ldp q1, q2, [x0]1403; VBITS_GE_128-NEXT: ldp q16, q5, [x0, #32]1404; VBITS_GE_128-NEXT: ldp q17, q6, [x1, #32]1405; VBITS_GE_128-NEXT: movprfx z4, z11406; VBITS_GE_128-NEXT: udiv z4.s, p0/m, z4.s, z0.s1407; VBITS_GE_128-NEXT: movprfx z19, z21408; VBITS_GE_128-NEXT: udiv z19.s, p0/m, z19.s, z3.s1409; VBITS_GE_128-NEXT: movprfx z7, z51410; VBITS_GE_128-NEXT: udiv z7.s, p0/m, z7.s, z6.s1411; VBITS_GE_128-NEXT: movprfx z18, z161412; VBITS_GE_128-NEXT: udiv z18.s, p0/m, z18.s, z17.s1413; VBITS_GE_128-NEXT: mls v1.4s, v4.4s, v0.4s1414; VBITS_GE_128-NEXT: mls v2.4s, v19.4s, v3.4s1415; VBITS_GE_128-NEXT: mls v16.4s, v18.4s, v17.4s1416; VBITS_GE_128-NEXT: mls v5.4s, v7.4s, v6.4s1417; VBITS_GE_128-NEXT: stp q1, q2, [x0]1418; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32]1419; VBITS_GE_128-NEXT: ret1420;1421; VBITS_GE_256-LABEL: urem_v16i32:1422; VBITS_GE_256: // %bb.0:1423; VBITS_GE_256-NEXT: ptrue p0.s, vl81424; VBITS_GE_256-NEXT: mov x8, #8 // =0x81425; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]1426; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]1427; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x0]1428; VBITS_GE_256-NEXT: ld1w { z4.s }, p0/z, [x1]1429; VBITS_GE_256-NEXT: movprfx z2, z01430; VBITS_GE_256-NEXT: udiv z2.s, p0/m, z2.s, z1.s1431; VBITS_GE_256-NEXT: movprfx z5, z31432; VBITS_GE_256-NEXT: udiv z5.s, p0/m, z5.s, z4.s1433; VBITS_GE_256-NEXT: mls z0.s, p0/m, z2.s, z1.s1434; VBITS_GE_256-NEXT: mls z3.s, p0/m, z5.s, z4.s1435; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]1436; VBITS_GE_256-NEXT: st1w { z3.s }, p0, [x0]1437; VBITS_GE_256-NEXT: ret1438;1439; VBITS_GE_512-LABEL: urem_v16i32:1440; VBITS_GE_512: // %bb.0:1441; VBITS_GE_512-NEXT: ptrue p0.s, vl161442; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]1443; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]1444; VBITS_GE_512-NEXT: movprfx z2, z01445; VBITS_GE_512-NEXT: udiv z2.s, p0/m, z2.s, z1.s1446; VBITS_GE_512-NEXT: mls z0.s, p0/m, z2.s, z1.s1447; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]1448; VBITS_GE_512-NEXT: ret1449 %op1 = load <16 x i32>, ptr %a1450 %op2 = load <16 x i32>, ptr %b1451 %res = urem <16 x i32> %op1, %op21452 store <16 x i32> %res, ptr %a1453 ret void1454}1455 1456define void @urem_v32i32(ptr %a, ptr %b) vscale_range(8,0) #0 {1457; CHECK-LABEL: urem_v32i32:1458; CHECK: // %bb.0:1459; CHECK-NEXT: ptrue p0.s, vl321460; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1461; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]1462; CHECK-NEXT: movprfx z2, z01463; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z1.s1464; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s1465; CHECK-NEXT: st1w { z0.s }, p0, [x0]1466; CHECK-NEXT: ret1467 %op1 = load <32 x i32>, ptr %a1468 %op2 = load <32 x i32>, ptr %b1469 %res = urem <32 x i32> %op1, %op21470 store <32 x i32> %res, ptr %a1471 ret void1472}1473 1474define void @urem_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {1475; CHECK-LABEL: urem_v64i32:1476; CHECK: // %bb.0:1477; CHECK-NEXT: ptrue p0.s, vl641478; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1479; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]1480; CHECK-NEXT: movprfx z2, z01481; CHECK-NEXT: udiv z2.s, p0/m, z2.s, z1.s1482; CHECK-NEXT: mls z0.s, p0/m, z2.s, z1.s1483; CHECK-NEXT: st1w { z0.s }, p0, [x0]1484; CHECK-NEXT: ret1485 %op1 = load <64 x i32>, ptr %a1486 %op2 = load <64 x i32>, ptr %b1487 %res = urem <64 x i32> %op1, %op21488 store <64 x i32> %res, ptr %a1489 ret void1490}1491 1492; Vector i64 udiv are not legal for NEON so use SVE when available.1493; FIXME: We should be able to improve the codegen for the 128 bits case here.1494define <1 x i64> @urem_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(1,0) #0 {1495; CHECK-LABEL: urem_v1i64:1496; CHECK: // %bb.0:1497; CHECK-NEXT: ptrue p0.d, vl11498; CHECK-NEXT: // kill: def $d1 killed $d1 def $z11499; CHECK-NEXT: // kill: def $d0 killed $d0 def $z01500; CHECK-NEXT: movprfx z2, z01501; CHECK-NEXT: udiv z2.d, p0/m, z2.d, z1.d1502; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d1503; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z01504; CHECK-NEXT: ret1505 %res = urem <1 x i64> %op1, %op21506 ret <1 x i64> %res1507}1508 1509; Vector i64 udiv are not legal for NEON so use SVE when available.1510; FIXME: We should be able to improve the codegen for the 128 bits case here.1511define <2 x i64> @urem_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(1,0) #0 {1512; CHECK-LABEL: urem_v2i64:1513; CHECK: // %bb.0:1514; CHECK-NEXT: ptrue p0.d, vl21515; CHECK-NEXT: // kill: def $q1 killed $q1 def $z11516; CHECK-NEXT: // kill: def $q0 killed $q0 def $z01517; CHECK-NEXT: movprfx z2, z01518; CHECK-NEXT: udiv z2.d, p0/m, z2.d, z1.d1519; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d1520; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z01521; CHECK-NEXT: ret1522 %res = urem <2 x i64> %op1, %op21523 ret <2 x i64> %res1524}1525 1526define void @urem_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {1527; CHECK-LABEL: urem_v4i64:1528; CHECK: // %bb.0:1529; CHECK-NEXT: ptrue p0.d, vl41530; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1531; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]1532; CHECK-NEXT: movprfx z2, z01533; CHECK-NEXT: udiv z2.d, p0/m, z2.d, z1.d1534; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d1535; CHECK-NEXT: st1d { z0.d }, p0, [x0]1536; CHECK-NEXT: ret1537 %op1 = load <4 x i64>, ptr %a1538 %op2 = load <4 x i64>, ptr %b1539 %res = urem <4 x i64> %op1, %op21540 store <4 x i64> %res, ptr %a1541 ret void1542}1543 1544define void @urem_v8i64(ptr %a, ptr %b) #0 {1545; VBITS_GE_128-LABEL: urem_v8i64:1546; VBITS_GE_128: // %bb.0:1547; VBITS_GE_128-NEXT: ldp q0, q3, [x1]1548; VBITS_GE_128-NEXT: ptrue p0.d, vl21549; VBITS_GE_128-NEXT: ldp q1, q2, [x0]1550; VBITS_GE_128-NEXT: ldp q16, q5, [x0, #32]1551; VBITS_GE_128-NEXT: ldp q17, q6, [x1, #32]1552; VBITS_GE_128-NEXT: movprfx z4, z11553; VBITS_GE_128-NEXT: udiv z4.d, p0/m, z4.d, z0.d1554; VBITS_GE_128-NEXT: movprfx z19, z21555; VBITS_GE_128-NEXT: udiv z19.d, p0/m, z19.d, z3.d1556; VBITS_GE_128-NEXT: movprfx z7, z51557; VBITS_GE_128-NEXT: udiv z7.d, p0/m, z7.d, z6.d1558; VBITS_GE_128-NEXT: movprfx z18, z161559; VBITS_GE_128-NEXT: udiv z18.d, p0/m, z18.d, z17.d1560; VBITS_GE_128-NEXT: msb z0.d, p0/m, z4.d, z1.d1561; VBITS_GE_128-NEXT: mls z2.d, p0/m, z19.d, z3.d1562; VBITS_GE_128-NEXT: mls z16.d, p0/m, z18.d, z17.d1563; VBITS_GE_128-NEXT: mls z5.d, p0/m, z7.d, z6.d1564; VBITS_GE_128-NEXT: stp q0, q2, [x0]1565; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32]1566; VBITS_GE_128-NEXT: ret1567;1568; VBITS_GE_256-LABEL: urem_v8i64:1569; VBITS_GE_256: // %bb.0:1570; VBITS_GE_256-NEXT: ptrue p0.d, vl41571; VBITS_GE_256-NEXT: mov x8, #4 // =0x41572; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]1573; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]1574; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x0]1575; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1]1576; VBITS_GE_256-NEXT: movprfx z2, z01577; VBITS_GE_256-NEXT: udiv z2.d, p0/m, z2.d, z1.d1578; VBITS_GE_256-NEXT: movprfx z5, z31579; VBITS_GE_256-NEXT: udiv z5.d, p0/m, z5.d, z4.d1580; VBITS_GE_256-NEXT: mls z0.d, p0/m, z2.d, z1.d1581; VBITS_GE_256-NEXT: mls z3.d, p0/m, z5.d, z4.d1582; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]1583; VBITS_GE_256-NEXT: st1d { z3.d }, p0, [x0]1584; VBITS_GE_256-NEXT: ret1585;1586; VBITS_GE_512-LABEL: urem_v8i64:1587; VBITS_GE_512: // %bb.0:1588; VBITS_GE_512-NEXT: ptrue p0.d, vl81589; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]1590; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]1591; VBITS_GE_512-NEXT: movprfx z2, z01592; VBITS_GE_512-NEXT: udiv z2.d, p0/m, z2.d, z1.d1593; VBITS_GE_512-NEXT: mls z0.d, p0/m, z2.d, z1.d1594; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]1595; VBITS_GE_512-NEXT: ret1596 %op1 = load <8 x i64>, ptr %a1597 %op2 = load <8 x i64>, ptr %b1598 %res = urem <8 x i64> %op1, %op21599 store <8 x i64> %res, ptr %a1600 ret void1601}1602 1603define void @urem_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 {1604; CHECK-LABEL: urem_v16i64:1605; CHECK: // %bb.0:1606; CHECK-NEXT: ptrue p0.d, vl161607; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1608; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]1609; CHECK-NEXT: movprfx z2, z01610; CHECK-NEXT: udiv z2.d, p0/m, z2.d, z1.d1611; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d1612; CHECK-NEXT: st1d { z0.d }, p0, [x0]1613; CHECK-NEXT: ret1614 %op1 = load <16 x i64>, ptr %a1615 %op2 = load <16 x i64>, ptr %b1616 %res = urem <16 x i64> %op1, %op21617 store <16 x i64> %res, ptr %a1618 ret void1619}1620 1621define void @urem_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {1622; CHECK-LABEL: urem_v32i64:1623; CHECK: // %bb.0:1624; CHECK-NEXT: ptrue p0.d, vl321625; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1626; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]1627; CHECK-NEXT: movprfx z2, z01628; CHECK-NEXT: udiv z2.d, p0/m, z2.d, z1.d1629; CHECK-NEXT: mls z0.d, p0/m, z2.d, z1.d1630; CHECK-NEXT: st1d { z0.d }, p0, [x0]1631; CHECK-NEXT: ret1632 %op1 = load <32 x i64>, ptr %a1633 %op2 = load <32 x i64>, ptr %b1634 %res = urem <32 x i64> %op1, %op21635 store <32 x i64> %res, ptr %a1636 ret void1637}1638 1639attributes #0 = { "target-features"="+sve" }1640