1213 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2563; RUN: llc -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5124; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5125 6target triple = "aarch64-unknown-linux-gnu"7 8;9; ASHR10;11 12; Don't use SVE for 64-bit vectors.13define <8 x i8> @ashr_v8i8(<8 x i8> %op1, <8 x i8> %op2) vscale_range(2,0) #0 {14; CHECK-LABEL: ashr_v8i8:15; CHECK: // %bb.0:16; CHECK-NEXT: neg v1.8b, v1.8b17; CHECK-NEXT: sshl v0.8b, v0.8b, v1.8b18; CHECK-NEXT: ret19 %res = ashr <8 x i8> %op1, %op220 ret <8 x i8> %res21}22 23; Don't use SVE for 128-bit vectors.24define <16 x i8> @ashr_v16i8(<16 x i8> %op1, <16 x i8> %op2) vscale_range(2,0) #0 {25; CHECK-LABEL: ashr_v16i8:26; CHECK: // %bb.0:27; CHECK-NEXT: neg v1.16b, v1.16b28; CHECK-NEXT: sshl v0.16b, v0.16b, v1.16b29; CHECK-NEXT: ret30 %res = ashr <16 x i8> %op1, %op231 ret <16 x i8> %res32}33 34define void @ashr_v32i8(ptr %a, ptr %b) vscale_range(2,0) #0 {35; CHECK-LABEL: ashr_v32i8:36; CHECK: // %bb.0:37; CHECK-NEXT: ptrue p0.b, vl3238; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]39; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]40; CHECK-NEXT: asr z0.b, p0/m, z0.b, z1.b41; CHECK-NEXT: st1b { z0.b }, p0, [x0]42; CHECK-NEXT: ret43 %op1 = load <32 x i8>, ptr %a44 %op2 = load <32 x i8>, ptr %b45 %res = ashr <32 x i8> %op1, %op246 store <32 x i8> %res, ptr %a47 ret void48}49 50define void @ashr_v64i8(ptr %a, ptr %b) #0 {51; VBITS_GE_256-LABEL: ashr_v64i8:52; VBITS_GE_256: // %bb.0:53; VBITS_GE_256-NEXT: ptrue p0.b, vl3254; VBITS_GE_256-NEXT: mov w8, #32 // =0x2055; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]56; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x1, x8]57; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0]58; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x1]59; VBITS_GE_256-NEXT: asr z0.b, p0/m, z0.b, z1.b60; VBITS_GE_256-NEXT: asr z2.b, p0/m, z2.b, z3.b61; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0, x8]62; VBITS_GE_256-NEXT: st1b { z2.b }, p0, [x0]63; VBITS_GE_256-NEXT: ret64;65; VBITS_GE_512-LABEL: ashr_v64i8:66; VBITS_GE_512: // %bb.0:67; VBITS_GE_512-NEXT: ptrue p0.b, vl6468; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]69; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x1]70; VBITS_GE_512-NEXT: asr z0.b, p0/m, z0.b, z1.b71; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0]72; VBITS_GE_512-NEXT: ret73 %op1 = load <64 x i8>, ptr %a74 %op2 = load <64 x i8>, ptr %b75 %res = ashr <64 x i8> %op1, %op276 store <64 x i8> %res, ptr %a77 ret void78}79 80define void @ashr_v128i8(ptr %a, ptr %b) vscale_range(8,0) #0 {81; CHECK-LABEL: ashr_v128i8:82; CHECK: // %bb.0:83; CHECK-NEXT: ptrue p0.b, vl12884; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]85; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]86; CHECK-NEXT: asr z0.b, p0/m, z0.b, z1.b87; CHECK-NEXT: st1b { z0.b }, p0, [x0]88; CHECK-NEXT: ret89 %op1 = load <128 x i8>, ptr %a90 %op2 = load <128 x i8>, ptr %b91 %res = ashr <128 x i8> %op1, %op292 store <128 x i8> %res, ptr %a93 ret void94}95 96define void @ashr_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {97; CHECK-LABEL: ashr_v256i8:98; CHECK: // %bb.0:99; CHECK-NEXT: ptrue p0.b, vl256100; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]101; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]102; CHECK-NEXT: asr z0.b, p0/m, z0.b, z1.b103; CHECK-NEXT: st1b { z0.b }, p0, [x0]104; CHECK-NEXT: ret105 %op1 = load <256 x i8>, ptr %a106 %op2 = load <256 x i8>, ptr %b107 %res = ashr <256 x i8> %op1, %op2108 store <256 x i8> %res, ptr %a109 ret void110}111 112; Don't use SVE for 64-bit vectors.113define <4 x i16> @ashr_v4i16(<4 x i16> %op1, <4 x i16> %op2) vscale_range(2,0) #0 {114; CHECK-LABEL: ashr_v4i16:115; CHECK: // %bb.0:116; CHECK-NEXT: neg v1.4h, v1.4h117; CHECK-NEXT: sshl v0.4h, v0.4h, v1.4h118; CHECK-NEXT: ret119 %res = ashr <4 x i16> %op1, %op2120 ret <4 x i16> %res121}122 123; Don't use SVE for 128-bit vectors.124define <8 x i16> @ashr_v8i16(<8 x i16> %op1, <8 x i16> %op2) vscale_range(2,0) #0 {125; CHECK-LABEL: ashr_v8i16:126; CHECK: // %bb.0:127; CHECK-NEXT: neg v1.8h, v1.8h128; CHECK-NEXT: sshl v0.8h, v0.8h, v1.8h129; CHECK-NEXT: ret130 %res = ashr <8 x i16> %op1, %op2131 ret <8 x i16> %res132}133 134define void @ashr_v16i16(ptr %a, ptr %b) vscale_range(2,0) #0 {135; CHECK-LABEL: ashr_v16i16:136; CHECK: // %bb.0:137; CHECK-NEXT: ptrue p0.h, vl16138; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]139; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]140; CHECK-NEXT: asr z0.h, p0/m, z0.h, z1.h141; CHECK-NEXT: st1h { z0.h }, p0, [x0]142; CHECK-NEXT: ret143 %op1 = load <16 x i16>, ptr %a144 %op2 = load <16 x i16>, ptr %b145 %res = ashr <16 x i16> %op1, %op2146 store <16 x i16> %res, ptr %a147 ret void148}149 150define void @ashr_v32i16(ptr %a, ptr %b) #0 {151; VBITS_GE_256-LABEL: ashr_v32i16:152; VBITS_GE_256: // %bb.0:153; VBITS_GE_256-NEXT: ptrue p0.h, vl16154; VBITS_GE_256-NEXT: mov x8, #16 // =0x10155; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]156; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1, x8, lsl #1]157; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0]158; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1]159; VBITS_GE_256-NEXT: asr z0.h, p0/m, z0.h, z1.h160; VBITS_GE_256-NEXT: asr z2.h, p0/m, z2.h, z3.h161; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]162; VBITS_GE_256-NEXT: st1h { z2.h }, p0, [x0]163; VBITS_GE_256-NEXT: ret164;165; VBITS_GE_512-LABEL: ashr_v32i16:166; VBITS_GE_512: // %bb.0:167; VBITS_GE_512-NEXT: ptrue p0.h, vl32168; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]169; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]170; VBITS_GE_512-NEXT: asr z0.h, p0/m, z0.h, z1.h171; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]172; VBITS_GE_512-NEXT: ret173 %op1 = load <32 x i16>, ptr %a174 %op2 = load <32 x i16>, ptr %b175 %res = ashr <32 x i16> %op1, %op2176 store <32 x i16> %res, ptr %a177 ret void178}179 180define void @ashr_v64i16(ptr %a, ptr %b) vscale_range(8,0) #0 {181; CHECK-LABEL: ashr_v64i16:182; CHECK: // %bb.0:183; CHECK-NEXT: ptrue p0.h, vl64184; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]185; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]186; CHECK-NEXT: asr z0.h, p0/m, z0.h, z1.h187; CHECK-NEXT: st1h { z0.h }, p0, [x0]188; CHECK-NEXT: ret189 %op1 = load <64 x i16>, ptr %a190 %op2 = load <64 x i16>, ptr %b191 %res = ashr <64 x i16> %op1, %op2192 store <64 x i16> %res, ptr %a193 ret void194}195 196define void @ashr_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {197; CHECK-LABEL: ashr_v128i16:198; CHECK: // %bb.0:199; CHECK-NEXT: ptrue p0.h, vl128200; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]201; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]202; CHECK-NEXT: asr z0.h, p0/m, z0.h, z1.h203; CHECK-NEXT: st1h { z0.h }, p0, [x0]204; CHECK-NEXT: ret205 %op1 = load <128 x i16>, ptr %a206 %op2 = load <128 x i16>, ptr %b207 %res = ashr <128 x i16> %op1, %op2208 store <128 x i16> %res, ptr %a209 ret void210}211 212; Don't use SVE for 64-bit vectors.213define <2 x i32> @ashr_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(2,0) #0 {214; CHECK-LABEL: ashr_v2i32:215; CHECK: // %bb.0:216; CHECK-NEXT: neg v1.2s, v1.2s217; CHECK-NEXT: sshl v0.2s, v0.2s, v1.2s218; CHECK-NEXT: ret219 %res = ashr <2 x i32> %op1, %op2220 ret <2 x i32> %res221}222 223; Don't use SVE for 128-bit vectors.224define <4 x i32> @ashr_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(2,0) #0 {225; CHECK-LABEL: ashr_v4i32:226; CHECK: // %bb.0:227; CHECK-NEXT: neg v1.4s, v1.4s228; CHECK-NEXT: sshl v0.4s, v0.4s, v1.4s229; CHECK-NEXT: ret230 %res = ashr <4 x i32> %op1, %op2231 ret <4 x i32> %res232}233 234define void @ashr_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {235; CHECK-LABEL: ashr_v8i32:236; CHECK: // %bb.0:237; CHECK-NEXT: ptrue p0.s, vl8238; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]239; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]240; CHECK-NEXT: asr z0.s, p0/m, z0.s, z1.s241; CHECK-NEXT: st1w { z0.s }, p0, [x0]242; CHECK-NEXT: ret243 %op1 = load <8 x i32>, ptr %a244 %op2 = load <8 x i32>, ptr %b245 %res = ashr <8 x i32> %op1, %op2246 store <8 x i32> %res, ptr %a247 ret void248}249 250define void @ashr_v16i32(ptr %a, ptr %b) #0 {251; VBITS_GE_256-LABEL: ashr_v16i32:252; VBITS_GE_256: // %bb.0:253; VBITS_GE_256-NEXT: ptrue p0.s, vl8254; VBITS_GE_256-NEXT: mov x8, #8 // =0x8255; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]256; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]257; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0]258; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1]259; VBITS_GE_256-NEXT: asr z0.s, p0/m, z0.s, z1.s260; VBITS_GE_256-NEXT: asr z2.s, p0/m, z2.s, z3.s261; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]262; VBITS_GE_256-NEXT: st1w { z2.s }, p0, [x0]263; VBITS_GE_256-NEXT: ret264;265; VBITS_GE_512-LABEL: ashr_v16i32:266; VBITS_GE_512: // %bb.0:267; VBITS_GE_512-NEXT: ptrue p0.s, vl16268; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]269; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]270; VBITS_GE_512-NEXT: asr z0.s, p0/m, z0.s, z1.s271; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]272; VBITS_GE_512-NEXT: ret273 %op1 = load <16 x i32>, ptr %a274 %op2 = load <16 x i32>, ptr %b275 %res = ashr <16 x i32> %op1, %op2276 store <16 x i32> %res, ptr %a277 ret void278}279 280define void @ashr_v32i32(ptr %a, ptr %b) vscale_range(8,0) #0 {281; CHECK-LABEL: ashr_v32i32:282; CHECK: // %bb.0:283; CHECK-NEXT: ptrue p0.s, vl32284; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]285; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]286; CHECK-NEXT: asr z0.s, p0/m, z0.s, z1.s287; CHECK-NEXT: st1w { z0.s }, p0, [x0]288; CHECK-NEXT: ret289 %op1 = load <32 x i32>, ptr %a290 %op2 = load <32 x i32>, ptr %b291 %res = ashr <32 x i32> %op1, %op2292 store <32 x i32> %res, ptr %a293 ret void294}295 296define void @ashr_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {297; CHECK-LABEL: ashr_v64i32:298; CHECK: // %bb.0:299; CHECK-NEXT: ptrue p0.s, vl64300; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]301; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]302; CHECK-NEXT: asr z0.s, p0/m, z0.s, z1.s303; CHECK-NEXT: st1w { z0.s }, p0, [x0]304; CHECK-NEXT: ret305 %op1 = load <64 x i32>, ptr %a306 %op2 = load <64 x i32>, ptr %b307 %res = ashr <64 x i32> %op1, %op2308 store <64 x i32> %res, ptr %a309 ret void310}311 312; Don't use SVE for 64-bit vectors.313define <1 x i64> @ashr_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(2,0) #0 {314; CHECK-LABEL: ashr_v1i64:315; CHECK: // %bb.0:316; CHECK-NEXT: neg d1, d1317; CHECK-NEXT: sshl d0, d0, d1318; CHECK-NEXT: ret319 %res = ashr <1 x i64> %op1, %op2320 ret <1 x i64> %res321}322 323; Don't use SVE for 128-bit vectors.324define <2 x i64> @ashr_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(2,0) #0 {325; CHECK-LABEL: ashr_v2i64:326; CHECK: // %bb.0:327; CHECK-NEXT: neg v1.2d, v1.2d328; CHECK-NEXT: sshl v0.2d, v0.2d, v1.2d329; CHECK-NEXT: ret330 %res = ashr <2 x i64> %op1, %op2331 ret <2 x i64> %res332}333 334define void @ashr_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {335; CHECK-LABEL: ashr_v4i64:336; CHECK: // %bb.0:337; CHECK-NEXT: ptrue p0.d, vl4338; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]339; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]340; CHECK-NEXT: asr z0.d, p0/m, z0.d, z1.d341; CHECK-NEXT: st1d { z0.d }, p0, [x0]342; CHECK-NEXT: ret343 %op1 = load <4 x i64>, ptr %a344 %op2 = load <4 x i64>, ptr %b345 %res = ashr <4 x i64> %op1, %op2346 store <4 x i64> %res, ptr %a347 ret void348}349 350define void @ashr_v8i64(ptr %a, ptr %b) #0 {351; VBITS_GE_256-LABEL: ashr_v8i64:352; VBITS_GE_256: // %bb.0:353; VBITS_GE_256-NEXT: ptrue p0.d, vl4354; VBITS_GE_256-NEXT: mov x8, #4 // =0x4355; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]356; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]357; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0]358; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1]359; VBITS_GE_256-NEXT: asr z0.d, p0/m, z0.d, z1.d360; VBITS_GE_256-NEXT: asr z2.d, p0/m, z2.d, z3.d361; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]362; VBITS_GE_256-NEXT: st1d { z2.d }, p0, [x0]363; VBITS_GE_256-NEXT: ret364;365; VBITS_GE_512-LABEL: ashr_v8i64:366; VBITS_GE_512: // %bb.0:367; VBITS_GE_512-NEXT: ptrue p0.d, vl8368; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]369; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]370; VBITS_GE_512-NEXT: asr z0.d, p0/m, z0.d, z1.d371; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]372; VBITS_GE_512-NEXT: ret373 %op1 = load <8 x i64>, ptr %a374 %op2 = load <8 x i64>, ptr %b375 %res = ashr <8 x i64> %op1, %op2376 store <8 x i64> %res, ptr %a377 ret void378}379 380define void @ashr_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 {381; CHECK-LABEL: ashr_v16i64:382; CHECK: // %bb.0:383; CHECK-NEXT: ptrue p0.d, vl16384; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]385; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]386; CHECK-NEXT: asr z0.d, p0/m, z0.d, z1.d387; CHECK-NEXT: st1d { z0.d }, p0, [x0]388; CHECK-NEXT: ret389 %op1 = load <16 x i64>, ptr %a390 %op2 = load <16 x i64>, ptr %b391 %res = ashr <16 x i64> %op1, %op2392 store <16 x i64> %res, ptr %a393 ret void394}395 396define void @ashr_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {397; CHECK-LABEL: ashr_v32i64:398; CHECK: // %bb.0:399; CHECK-NEXT: ptrue p0.d, vl32400; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]401; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]402; CHECK-NEXT: asr z0.d, p0/m, z0.d, z1.d403; CHECK-NEXT: st1d { z0.d }, p0, [x0]404; CHECK-NEXT: ret405 %op1 = load <32 x i64>, ptr %a406 %op2 = load <32 x i64>, ptr %b407 %res = ashr <32 x i64> %op1, %op2408 store <32 x i64> %res, ptr %a409 ret void410}411 412;413; LSHR414;415 416; Don't use SVE for 64-bit vectors.417define <8 x i8> @lshr_v8i8(<8 x i8> %op1, <8 x i8> %op2) vscale_range(2,0) #0 {418; CHECK-LABEL: lshr_v8i8:419; CHECK: // %bb.0:420; CHECK-NEXT: neg v1.8b, v1.8b421; CHECK-NEXT: ushl v0.8b, v0.8b, v1.8b422; CHECK-NEXT: ret423 %res = lshr <8 x i8> %op1, %op2424 ret <8 x i8> %res425}426 427; Don't use SVE for 128-bit vectors.428define <16 x i8> @lshr_v16i8(<16 x i8> %op1, <16 x i8> %op2) vscale_range(2,0) #0 {429; CHECK-LABEL: lshr_v16i8:430; CHECK: // %bb.0:431; CHECK-NEXT: neg v1.16b, v1.16b432; CHECK-NEXT: ushl v0.16b, v0.16b, v1.16b433; CHECK-NEXT: ret434 %res = lshr <16 x i8> %op1, %op2435 ret <16 x i8> %res436}437 438define void @lshr_v32i8(ptr %a, ptr %b) vscale_range(2,0) #0 {439; CHECK-LABEL: lshr_v32i8:440; CHECK: // %bb.0:441; CHECK-NEXT: ptrue p0.b, vl32442; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]443; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]444; CHECK-NEXT: lsr z0.b, p0/m, z0.b, z1.b445; CHECK-NEXT: st1b { z0.b }, p0, [x0]446; CHECK-NEXT: ret447 %op1 = load <32 x i8>, ptr %a448 %op2 = load <32 x i8>, ptr %b449 %res = lshr <32 x i8> %op1, %op2450 store <32 x i8> %res, ptr %a451 ret void452}453 454define void @lshr_v64i8(ptr %a, ptr %b) #0 {455; VBITS_GE_256-LABEL: lshr_v64i8:456; VBITS_GE_256: // %bb.0:457; VBITS_GE_256-NEXT: ptrue p0.b, vl32458; VBITS_GE_256-NEXT: mov w8, #32 // =0x20459; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]460; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x1, x8]461; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0]462; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x1]463; VBITS_GE_256-NEXT: lsr z0.b, p0/m, z0.b, z1.b464; VBITS_GE_256-NEXT: lsr z2.b, p0/m, z2.b, z3.b465; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0, x8]466; VBITS_GE_256-NEXT: st1b { z2.b }, p0, [x0]467; VBITS_GE_256-NEXT: ret468;469; VBITS_GE_512-LABEL: lshr_v64i8:470; VBITS_GE_512: // %bb.0:471; VBITS_GE_512-NEXT: ptrue p0.b, vl64472; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]473; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x1]474; VBITS_GE_512-NEXT: lsr z0.b, p0/m, z0.b, z1.b475; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0]476; VBITS_GE_512-NEXT: ret477 %op1 = load <64 x i8>, ptr %a478 %op2 = load <64 x i8>, ptr %b479 %res = lshr <64 x i8> %op1, %op2480 store <64 x i8> %res, ptr %a481 ret void482}483 484define void @lshr_v128i8(ptr %a, ptr %b) vscale_range(8,0) #0 {485; CHECK-LABEL: lshr_v128i8:486; CHECK: // %bb.0:487; CHECK-NEXT: ptrue p0.b, vl128488; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]489; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]490; CHECK-NEXT: lsr z0.b, p0/m, z0.b, z1.b491; CHECK-NEXT: st1b { z0.b }, p0, [x0]492; CHECK-NEXT: ret493 %op1 = load <128 x i8>, ptr %a494 %op2 = load <128 x i8>, ptr %b495 %res = lshr <128 x i8> %op1, %op2496 store <128 x i8> %res, ptr %a497 ret void498}499 500define void @lshr_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {501; CHECK-LABEL: lshr_v256i8:502; CHECK: // %bb.0:503; CHECK-NEXT: ptrue p0.b, vl256504; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]505; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]506; CHECK-NEXT: lsr z0.b, p0/m, z0.b, z1.b507; CHECK-NEXT: st1b { z0.b }, p0, [x0]508; CHECK-NEXT: ret509 %op1 = load <256 x i8>, ptr %a510 %op2 = load <256 x i8>, ptr %b511 %res = lshr <256 x i8> %op1, %op2512 store <256 x i8> %res, ptr %a513 ret void514}515 516; Don't use SVE for 64-bit vectors.517define <4 x i16> @lshr_v4i16(<4 x i16> %op1, <4 x i16> %op2) vscale_range(2,0) #0 {518; CHECK-LABEL: lshr_v4i16:519; CHECK: // %bb.0:520; CHECK-NEXT: neg v1.4h, v1.4h521; CHECK-NEXT: ushl v0.4h, v0.4h, v1.4h522; CHECK-NEXT: ret523 %res = lshr <4 x i16> %op1, %op2524 ret <4 x i16> %res525}526 527; Don't use SVE for 128-bit vectors.528define <8 x i16> @lshr_v8i16(<8 x i16> %op1, <8 x i16> %op2) vscale_range(2,0) #0 {529; CHECK-LABEL: lshr_v8i16:530; CHECK: // %bb.0:531; CHECK-NEXT: neg v1.8h, v1.8h532; CHECK-NEXT: ushl v0.8h, v0.8h, v1.8h533; CHECK-NEXT: ret534 %res = lshr <8 x i16> %op1, %op2535 ret <8 x i16> %res536}537 538define void @lshr_v16i16(ptr %a, ptr %b) vscale_range(2,0) #0 {539; CHECK-LABEL: lshr_v16i16:540; CHECK: // %bb.0:541; CHECK-NEXT: ptrue p0.h, vl16542; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]543; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]544; CHECK-NEXT: lsr z0.h, p0/m, z0.h, z1.h545; CHECK-NEXT: st1h { z0.h }, p0, [x0]546; CHECK-NEXT: ret547 %op1 = load <16 x i16>, ptr %a548 %op2 = load <16 x i16>, ptr %b549 %res = lshr <16 x i16> %op1, %op2550 store <16 x i16> %res, ptr %a551 ret void552}553 554define void @lshr_v32i16(ptr %a, ptr %b) #0 {555; VBITS_GE_256-LABEL: lshr_v32i16:556; VBITS_GE_256: // %bb.0:557; VBITS_GE_256-NEXT: ptrue p0.h, vl16558; VBITS_GE_256-NEXT: mov x8, #16 // =0x10559; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]560; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1, x8, lsl #1]561; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0]562; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1]563; VBITS_GE_256-NEXT: lsr z0.h, p0/m, z0.h, z1.h564; VBITS_GE_256-NEXT: lsr z2.h, p0/m, z2.h, z3.h565; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]566; VBITS_GE_256-NEXT: st1h { z2.h }, p0, [x0]567; VBITS_GE_256-NEXT: ret568;569; VBITS_GE_512-LABEL: lshr_v32i16:570; VBITS_GE_512: // %bb.0:571; VBITS_GE_512-NEXT: ptrue p0.h, vl32572; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]573; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]574; VBITS_GE_512-NEXT: lsr z0.h, p0/m, z0.h, z1.h575; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]576; VBITS_GE_512-NEXT: ret577 %op1 = load <32 x i16>, ptr %a578 %op2 = load <32 x i16>, ptr %b579 %res = lshr <32 x i16> %op1, %op2580 store <32 x i16> %res, ptr %a581 ret void582}583 584define void @lshr_v64i16(ptr %a, ptr %b) vscale_range(8,0) #0 {585; CHECK-LABEL: lshr_v64i16:586; CHECK: // %bb.0:587; CHECK-NEXT: ptrue p0.h, vl64588; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]589; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]590; CHECK-NEXT: lsr z0.h, p0/m, z0.h, z1.h591; CHECK-NEXT: st1h { z0.h }, p0, [x0]592; CHECK-NEXT: ret593 %op1 = load <64 x i16>, ptr %a594 %op2 = load <64 x i16>, ptr %b595 %res = lshr <64 x i16> %op1, %op2596 store <64 x i16> %res, ptr %a597 ret void598}599 600define void @lshr_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {601; CHECK-LABEL: lshr_v128i16:602; CHECK: // %bb.0:603; CHECK-NEXT: ptrue p0.h, vl128604; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]605; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]606; CHECK-NEXT: lsr z0.h, p0/m, z0.h, z1.h607; CHECK-NEXT: st1h { z0.h }, p0, [x0]608; CHECK-NEXT: ret609 %op1 = load <128 x i16>, ptr %a610 %op2 = load <128 x i16>, ptr %b611 %res = lshr <128 x i16> %op1, %op2612 store <128 x i16> %res, ptr %a613 ret void614}615 616; Don't use SVE for 64-bit vectors.617define <2 x i32> @lshr_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(2,0) #0 {618; CHECK-LABEL: lshr_v2i32:619; CHECK: // %bb.0:620; CHECK-NEXT: neg v1.2s, v1.2s621; CHECK-NEXT: ushl v0.2s, v0.2s, v1.2s622; CHECK-NEXT: ret623 %res = lshr <2 x i32> %op1, %op2624 ret <2 x i32> %res625}626 627; Don't use SVE for 128-bit vectors.628define <4 x i32> @lshr_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(2,0) #0 {629; CHECK-LABEL: lshr_v4i32:630; CHECK: // %bb.0:631; CHECK-NEXT: neg v1.4s, v1.4s632; CHECK-NEXT: ushl v0.4s, v0.4s, v1.4s633; CHECK-NEXT: ret634 %res = lshr <4 x i32> %op1, %op2635 ret <4 x i32> %res636}637 638define void @lshr_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {639; CHECK-LABEL: lshr_v8i32:640; CHECK: // %bb.0:641; CHECK-NEXT: ptrue p0.s, vl8642; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]643; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]644; CHECK-NEXT: lsr z0.s, p0/m, z0.s, z1.s645; CHECK-NEXT: st1w { z0.s }, p0, [x0]646; CHECK-NEXT: ret647 %op1 = load <8 x i32>, ptr %a648 %op2 = load <8 x i32>, ptr %b649 %res = lshr <8 x i32> %op1, %op2650 store <8 x i32> %res, ptr %a651 ret void652}653 654define void @lshr_v16i32(ptr %a, ptr %b) #0 {655; VBITS_GE_256-LABEL: lshr_v16i32:656; VBITS_GE_256: // %bb.0:657; VBITS_GE_256-NEXT: ptrue p0.s, vl8658; VBITS_GE_256-NEXT: mov x8, #8 // =0x8659; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]660; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]661; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0]662; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1]663; VBITS_GE_256-NEXT: lsr z0.s, p0/m, z0.s, z1.s664; VBITS_GE_256-NEXT: lsr z2.s, p0/m, z2.s, z3.s665; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]666; VBITS_GE_256-NEXT: st1w { z2.s }, p0, [x0]667; VBITS_GE_256-NEXT: ret668;669; VBITS_GE_512-LABEL: lshr_v16i32:670; VBITS_GE_512: // %bb.0:671; VBITS_GE_512-NEXT: ptrue p0.s, vl16672; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]673; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]674; VBITS_GE_512-NEXT: lsr z0.s, p0/m, z0.s, z1.s675; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]676; VBITS_GE_512-NEXT: ret677 %op1 = load <16 x i32>, ptr %a678 %op2 = load <16 x i32>, ptr %b679 %res = lshr <16 x i32> %op1, %op2680 store <16 x i32> %res, ptr %a681 ret void682}683 684define void @lshr_v32i32(ptr %a, ptr %b) vscale_range(8,0) #0 {685; CHECK-LABEL: lshr_v32i32:686; CHECK: // %bb.0:687; CHECK-NEXT: ptrue p0.s, vl32688; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]689; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]690; CHECK-NEXT: lsr z0.s, p0/m, z0.s, z1.s691; CHECK-NEXT: st1w { z0.s }, p0, [x0]692; CHECK-NEXT: ret693 %op1 = load <32 x i32>, ptr %a694 %op2 = load <32 x i32>, ptr %b695 %res = lshr <32 x i32> %op1, %op2696 store <32 x i32> %res, ptr %a697 ret void698}699 700define void @lshr_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {701; CHECK-LABEL: lshr_v64i32:702; CHECK: // %bb.0:703; CHECK-NEXT: ptrue p0.s, vl64704; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]705; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]706; CHECK-NEXT: lsr z0.s, p0/m, z0.s, z1.s707; CHECK-NEXT: st1w { z0.s }, p0, [x0]708; CHECK-NEXT: ret709 %op1 = load <64 x i32>, ptr %a710 %op2 = load <64 x i32>, ptr %b711 %res = lshr <64 x i32> %op1, %op2712 store <64 x i32> %res, ptr %a713 ret void714}715 716; Don't use SVE for 64-bit vectors.717define <1 x i64> @lshr_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(2,0) #0 {718; CHECK-LABEL: lshr_v1i64:719; CHECK: // %bb.0:720; CHECK-NEXT: neg d1, d1721; CHECK-NEXT: ushl d0, d0, d1722; CHECK-NEXT: ret723 %res = lshr <1 x i64> %op1, %op2724 ret <1 x i64> %res725}726 727; Don't use SVE for 128-bit vectors.728define <2 x i64> @lshr_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(2,0) #0 {729; CHECK-LABEL: lshr_v2i64:730; CHECK: // %bb.0:731; CHECK-NEXT: neg v1.2d, v1.2d732; CHECK-NEXT: ushl v0.2d, v0.2d, v1.2d733; CHECK-NEXT: ret734 %res = lshr <2 x i64> %op1, %op2735 ret <2 x i64> %res736}737 738define void @lshr_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {739; CHECK-LABEL: lshr_v4i64:740; CHECK: // %bb.0:741; CHECK-NEXT: ptrue p0.d, vl4742; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]743; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]744; CHECK-NEXT: lsr z0.d, p0/m, z0.d, z1.d745; CHECK-NEXT: st1d { z0.d }, p0, [x0]746; CHECK-NEXT: ret747 %op1 = load <4 x i64>, ptr %a748 %op2 = load <4 x i64>, ptr %b749 %res = lshr <4 x i64> %op1, %op2750 store <4 x i64> %res, ptr %a751 ret void752}753 754define void @lshr_v8i64(ptr %a, ptr %b) #0 {755; VBITS_GE_256-LABEL: lshr_v8i64:756; VBITS_GE_256: // %bb.0:757; VBITS_GE_256-NEXT: ptrue p0.d, vl4758; VBITS_GE_256-NEXT: mov x8, #4 // =0x4759; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]760; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]761; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0]762; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1]763; VBITS_GE_256-NEXT: lsr z0.d, p0/m, z0.d, z1.d764; VBITS_GE_256-NEXT: lsr z2.d, p0/m, z2.d, z3.d765; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]766; VBITS_GE_256-NEXT: st1d { z2.d }, p0, [x0]767; VBITS_GE_256-NEXT: ret768;769; VBITS_GE_512-LABEL: lshr_v8i64:770; VBITS_GE_512: // %bb.0:771; VBITS_GE_512-NEXT: ptrue p0.d, vl8772; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]773; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]774; VBITS_GE_512-NEXT: lsr z0.d, p0/m, z0.d, z1.d775; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]776; VBITS_GE_512-NEXT: ret777 %op1 = load <8 x i64>, ptr %a778 %op2 = load <8 x i64>, ptr %b779 %res = lshr <8 x i64> %op1, %op2780 store <8 x i64> %res, ptr %a781 ret void782}783 784define void @lshr_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 {785; CHECK-LABEL: lshr_v16i64:786; CHECK: // %bb.0:787; CHECK-NEXT: ptrue p0.d, vl16788; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]789; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]790; CHECK-NEXT: lsr z0.d, p0/m, z0.d, z1.d791; CHECK-NEXT: st1d { z0.d }, p0, [x0]792; CHECK-NEXT: ret793 %op1 = load <16 x i64>, ptr %a794 %op2 = load <16 x i64>, ptr %b795 %res = lshr <16 x i64> %op1, %op2796 store <16 x i64> %res, ptr %a797 ret void798}799 800define void @lshr_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {801; CHECK-LABEL: lshr_v32i64:802; CHECK: // %bb.0:803; CHECK-NEXT: ptrue p0.d, vl32804; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]805; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]806; CHECK-NEXT: lsr z0.d, p0/m, z0.d, z1.d807; CHECK-NEXT: st1d { z0.d }, p0, [x0]808; CHECK-NEXT: ret809 %op1 = load <32 x i64>, ptr %a810 %op2 = load <32 x i64>, ptr %b811 %res = lshr <32 x i64> %op1, %op2812 store <32 x i64> %res, ptr %a813 ret void814}815 816;817; SHL818;819 820; Don't use SVE for 64-bit vectors.821define <8 x i8> @shl_v8i8(<8 x i8> %op1, <8 x i8> %op2) vscale_range(2,0) #0 {822; CHECK-LABEL: shl_v8i8:823; CHECK: // %bb.0:824; CHECK-NEXT: ushl v0.8b, v0.8b, v1.8b825; CHECK-NEXT: ret826 %res = shl <8 x i8> %op1, %op2827 ret <8 x i8> %res828}829 830; Don't use SVE for 128-bit vectors.831define <16 x i8> @shl_v16i8(<16 x i8> %op1, <16 x i8> %op2) vscale_range(2,0) #0 {832; CHECK-LABEL: shl_v16i8:833; CHECK: // %bb.0:834; CHECK-NEXT: ushl v0.16b, v0.16b, v1.16b835; CHECK-NEXT: ret836 %res = shl <16 x i8> %op1, %op2837 ret <16 x i8> %res838}839 840define void @shl_v32i8(ptr %a, ptr %b) vscale_range(2,0) #0 {841; CHECK-LABEL: shl_v32i8:842; CHECK: // %bb.0:843; CHECK-NEXT: ptrue p0.b, vl32844; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]845; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]846; CHECK-NEXT: lsl z0.b, p0/m, z0.b, z1.b847; CHECK-NEXT: st1b { z0.b }, p0, [x0]848; CHECK-NEXT: ret849 %op1 = load <32 x i8>, ptr %a850 %op2 = load <32 x i8>, ptr %b851 %res = shl <32 x i8> %op1, %op2852 store <32 x i8> %res, ptr %a853 ret void854}855 856define void @shl_v64i8(ptr %a, ptr %b) #0 {857; VBITS_GE_256-LABEL: shl_v64i8:858; VBITS_GE_256: // %bb.0:859; VBITS_GE_256-NEXT: ptrue p0.b, vl32860; VBITS_GE_256-NEXT: mov w8, #32 // =0x20861; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]862; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x1, x8]863; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0]864; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x1]865; VBITS_GE_256-NEXT: lsl z0.b, p0/m, z0.b, z1.b866; VBITS_GE_256-NEXT: lsl z2.b, p0/m, z2.b, z3.b867; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0, x8]868; VBITS_GE_256-NEXT: st1b { z2.b }, p0, [x0]869; VBITS_GE_256-NEXT: ret870;871; VBITS_GE_512-LABEL: shl_v64i8:872; VBITS_GE_512: // %bb.0:873; VBITS_GE_512-NEXT: ptrue p0.b, vl64874; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]875; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x1]876; VBITS_GE_512-NEXT: lsl z0.b, p0/m, z0.b, z1.b877; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0]878; VBITS_GE_512-NEXT: ret879 %op1 = load <64 x i8>, ptr %a880 %op2 = load <64 x i8>, ptr %b881 %res = shl <64 x i8> %op1, %op2882 store <64 x i8> %res, ptr %a883 ret void884}885 886define void @shl_v128i8(ptr %a, ptr %b) vscale_range(8,0) #0 {887; CHECK-LABEL: shl_v128i8:888; CHECK: // %bb.0:889; CHECK-NEXT: ptrue p0.b, vl128890; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]891; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]892; CHECK-NEXT: lsl z0.b, p0/m, z0.b, z1.b893; CHECK-NEXT: st1b { z0.b }, p0, [x0]894; CHECK-NEXT: ret895 %op1 = load <128 x i8>, ptr %a896 %op2 = load <128 x i8>, ptr %b897 %res = shl <128 x i8> %op1, %op2898 store <128 x i8> %res, ptr %a899 ret void900}901 902define void @shl_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {903; CHECK-LABEL: shl_v256i8:904; CHECK: // %bb.0:905; CHECK-NEXT: ptrue p0.b, vl256906; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]907; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1]908; CHECK-NEXT: lsl z0.b, p0/m, z0.b, z1.b909; CHECK-NEXT: st1b { z0.b }, p0, [x0]910; CHECK-NEXT: ret911 %op1 = load <256 x i8>, ptr %a912 %op2 = load <256 x i8>, ptr %b913 %res = shl <256 x i8> %op1, %op2914 store <256 x i8> %res, ptr %a915 ret void916}917 918; Don't use SVE for 64-bit vectors.919define <4 x i16> @shl_v4i16(<4 x i16> %op1, <4 x i16> %op2) vscale_range(2,0) #0 {920; CHECK-LABEL: shl_v4i16:921; CHECK: // %bb.0:922; CHECK-NEXT: ushl v0.4h, v0.4h, v1.4h923; CHECK-NEXT: ret924 %res = shl <4 x i16> %op1, %op2925 ret <4 x i16> %res926}927 928; Don't use SVE for 128-bit vectors.929define <8 x i16> @shl_v8i16(<8 x i16> %op1, <8 x i16> %op2) vscale_range(2,0) #0 {930; CHECK-LABEL: shl_v8i16:931; CHECK: // %bb.0:932; CHECK-NEXT: ushl v0.8h, v0.8h, v1.8h933; CHECK-NEXT: ret934 %res = shl <8 x i16> %op1, %op2935 ret <8 x i16> %res936}937 938define void @shl_v16i16(ptr %a, ptr %b) vscale_range(2,0) #0 {939; CHECK-LABEL: shl_v16i16:940; CHECK: // %bb.0:941; CHECK-NEXT: ptrue p0.h, vl16942; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]943; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]944; CHECK-NEXT: lsl z0.h, p0/m, z0.h, z1.h945; CHECK-NEXT: st1h { z0.h }, p0, [x0]946; CHECK-NEXT: ret947 %op1 = load <16 x i16>, ptr %a948 %op2 = load <16 x i16>, ptr %b949 %res = shl <16 x i16> %op1, %op2950 store <16 x i16> %res, ptr %a951 ret void952}953 954define void @shl_v32i16(ptr %a, ptr %b) #0 {955; VBITS_GE_256-LABEL: shl_v32i16:956; VBITS_GE_256: // %bb.0:957; VBITS_GE_256-NEXT: ptrue p0.h, vl16958; VBITS_GE_256-NEXT: mov x8, #16 // =0x10959; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]960; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1, x8, lsl #1]961; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0]962; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1]963; VBITS_GE_256-NEXT: lsl z0.h, p0/m, z0.h, z1.h964; VBITS_GE_256-NEXT: lsl z2.h, p0/m, z2.h, z3.h965; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]966; VBITS_GE_256-NEXT: st1h { z2.h }, p0, [x0]967; VBITS_GE_256-NEXT: ret968;969; VBITS_GE_512-LABEL: shl_v32i16:970; VBITS_GE_512: // %bb.0:971; VBITS_GE_512-NEXT: ptrue p0.h, vl32972; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]973; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]974; VBITS_GE_512-NEXT: lsl z0.h, p0/m, z0.h, z1.h975; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]976; VBITS_GE_512-NEXT: ret977 %op1 = load <32 x i16>, ptr %a978 %op2 = load <32 x i16>, ptr %b979 %res = shl <32 x i16> %op1, %op2980 store <32 x i16> %res, ptr %a981 ret void982}983 984define void @shl_v64i16(ptr %a, ptr %b) vscale_range(8,0) #0 {985; CHECK-LABEL: shl_v64i16:986; CHECK: // %bb.0:987; CHECK-NEXT: ptrue p0.h, vl64988; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]989; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]990; CHECK-NEXT: lsl z0.h, p0/m, z0.h, z1.h991; CHECK-NEXT: st1h { z0.h }, p0, [x0]992; CHECK-NEXT: ret993 %op1 = load <64 x i16>, ptr %a994 %op2 = load <64 x i16>, ptr %b995 %res = shl <64 x i16> %op1, %op2996 store <64 x i16> %res, ptr %a997 ret void998}999 1000define void @shl_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {1001; CHECK-LABEL: shl_v128i16:1002; CHECK: // %bb.0:1003; CHECK-NEXT: ptrue p0.h, vl1281004; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1005; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]1006; CHECK-NEXT: lsl z0.h, p0/m, z0.h, z1.h1007; CHECK-NEXT: st1h { z0.h }, p0, [x0]1008; CHECK-NEXT: ret1009 %op1 = load <128 x i16>, ptr %a1010 %op2 = load <128 x i16>, ptr %b1011 %res = shl <128 x i16> %op1, %op21012 store <128 x i16> %res, ptr %a1013 ret void1014}1015 1016; Don't use SVE for 64-bit vectors.1017define <2 x i32> @shl_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(2,0) #0 {1018; CHECK-LABEL: shl_v2i32:1019; CHECK: // %bb.0:1020; CHECK-NEXT: ushl v0.2s, v0.2s, v1.2s1021; CHECK-NEXT: ret1022 %res = shl <2 x i32> %op1, %op21023 ret <2 x i32> %res1024}1025 1026; Don't use SVE for 128-bit vectors.1027define <4 x i32> @shl_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(2,0) #0 {1028; CHECK-LABEL: shl_v4i32:1029; CHECK: // %bb.0:1030; CHECK-NEXT: ushl v0.4s, v0.4s, v1.4s1031; CHECK-NEXT: ret1032 %res = shl <4 x i32> %op1, %op21033 ret <4 x i32> %res1034}1035 1036define void @shl_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {1037; CHECK-LABEL: shl_v8i32:1038; CHECK: // %bb.0:1039; CHECK-NEXT: ptrue p0.s, vl81040; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1041; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]1042; CHECK-NEXT: lsl z0.s, p0/m, z0.s, z1.s1043; CHECK-NEXT: st1w { z0.s }, p0, [x0]1044; CHECK-NEXT: ret1045 %op1 = load <8 x i32>, ptr %a1046 %op2 = load <8 x i32>, ptr %b1047 %res = shl <8 x i32> %op1, %op21048 store <8 x i32> %res, ptr %a1049 ret void1050}1051 1052define void @shl_v16i32(ptr %a, ptr %b) #0 {1053; VBITS_GE_256-LABEL: shl_v16i32:1054; VBITS_GE_256: // %bb.0:1055; VBITS_GE_256-NEXT: ptrue p0.s, vl81056; VBITS_GE_256-NEXT: mov x8, #8 // =0x81057; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]1058; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]1059; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0]1060; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1]1061; VBITS_GE_256-NEXT: lsl z0.s, p0/m, z0.s, z1.s1062; VBITS_GE_256-NEXT: lsl z2.s, p0/m, z2.s, z3.s1063; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]1064; VBITS_GE_256-NEXT: st1w { z2.s }, p0, [x0]1065; VBITS_GE_256-NEXT: ret1066;1067; VBITS_GE_512-LABEL: shl_v16i32:1068; VBITS_GE_512: // %bb.0:1069; VBITS_GE_512-NEXT: ptrue p0.s, vl161070; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]1071; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]1072; VBITS_GE_512-NEXT: lsl z0.s, p0/m, z0.s, z1.s1073; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]1074; VBITS_GE_512-NEXT: ret1075 %op1 = load <16 x i32>, ptr %a1076 %op2 = load <16 x i32>, ptr %b1077 %res = shl <16 x i32> %op1, %op21078 store <16 x i32> %res, ptr %a1079 ret void1080}1081 1082define void @shl_v32i32(ptr %a, ptr %b) vscale_range(8,0) #0 {1083; CHECK-LABEL: shl_v32i32:1084; CHECK: // %bb.0:1085; CHECK-NEXT: ptrue p0.s, vl321086; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1087; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]1088; CHECK-NEXT: lsl z0.s, p0/m, z0.s, z1.s1089; CHECK-NEXT: st1w { z0.s }, p0, [x0]1090; CHECK-NEXT: ret1091 %op1 = load <32 x i32>, ptr %a1092 %op2 = load <32 x i32>, ptr %b1093 %res = shl <32 x i32> %op1, %op21094 store <32 x i32> %res, ptr %a1095 ret void1096}1097 1098define void @shl_v64i32(ptr %a, ptr %b) vscale_range(16,0) #0 {1099; CHECK-LABEL: shl_v64i32:1100; CHECK: // %bb.0:1101; CHECK-NEXT: ptrue p0.s, vl641102; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1103; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]1104; CHECK-NEXT: lsl z0.s, p0/m, z0.s, z1.s1105; CHECK-NEXT: st1w { z0.s }, p0, [x0]1106; CHECK-NEXT: ret1107 %op1 = load <64 x i32>, ptr %a1108 %op2 = load <64 x i32>, ptr %b1109 %res = shl <64 x i32> %op1, %op21110 store <64 x i32> %res, ptr %a1111 ret void1112}1113 1114; Don't use SVE for 64-bit vectors.1115define <1 x i64> @shl_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(2,0) #0 {1116; CHECK-LABEL: shl_v1i64:1117; CHECK: // %bb.0:1118; CHECK-NEXT: ushl d0, d0, d11119; CHECK-NEXT: ret1120 %res = shl <1 x i64> %op1, %op21121 ret <1 x i64> %res1122}1123 1124; Don't use SVE for 128-bit vectors.1125define <2 x i64> @shl_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(2,0) #0 {1126; CHECK-LABEL: shl_v2i64:1127; CHECK: // %bb.0:1128; CHECK-NEXT: ushl v0.2d, v0.2d, v1.2d1129; CHECK-NEXT: ret1130 %res = shl <2 x i64> %op1, %op21131 ret <2 x i64> %res1132}1133 1134define void @shl_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {1135; CHECK-LABEL: shl_v4i64:1136; CHECK: // %bb.0:1137; CHECK-NEXT: ptrue p0.d, vl41138; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1139; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]1140; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z1.d1141; CHECK-NEXT: st1d { z0.d }, p0, [x0]1142; CHECK-NEXT: ret1143 %op1 = load <4 x i64>, ptr %a1144 %op2 = load <4 x i64>, ptr %b1145 %res = shl <4 x i64> %op1, %op21146 store <4 x i64> %res, ptr %a1147 ret void1148}1149 1150define void @shl_v8i64(ptr %a, ptr %b) #0 {1151; VBITS_GE_256-LABEL: shl_v8i64:1152; VBITS_GE_256: // %bb.0:1153; VBITS_GE_256-NEXT: ptrue p0.d, vl41154; VBITS_GE_256-NEXT: mov x8, #4 // =0x41155; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]1156; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]1157; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0]1158; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1]1159; VBITS_GE_256-NEXT: lsl z0.d, p0/m, z0.d, z1.d1160; VBITS_GE_256-NEXT: lsl z2.d, p0/m, z2.d, z3.d1161; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]1162; VBITS_GE_256-NEXT: st1d { z2.d }, p0, [x0]1163; VBITS_GE_256-NEXT: ret1164;1165; VBITS_GE_512-LABEL: shl_v8i64:1166; VBITS_GE_512: // %bb.0:1167; VBITS_GE_512-NEXT: ptrue p0.d, vl81168; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]1169; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]1170; VBITS_GE_512-NEXT: lsl z0.d, p0/m, z0.d, z1.d1171; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]1172; VBITS_GE_512-NEXT: ret1173 %op1 = load <8 x i64>, ptr %a1174 %op2 = load <8 x i64>, ptr %b1175 %res = shl <8 x i64> %op1, %op21176 store <8 x i64> %res, ptr %a1177 ret void1178}1179 1180define void @shl_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 {1181; CHECK-LABEL: shl_v16i64:1182; CHECK: // %bb.0:1183; CHECK-NEXT: ptrue p0.d, vl161184; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1185; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]1186; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z1.d1187; CHECK-NEXT: st1d { z0.d }, p0, [x0]1188; CHECK-NEXT: ret1189 %op1 = load <16 x i64>, ptr %a1190 %op2 = load <16 x i64>, ptr %b1191 %res = shl <16 x i64> %op1, %op21192 store <16 x i64> %res, ptr %a1193 ret void1194}1195 1196define void @shl_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {1197; CHECK-LABEL: shl_v32i64:1198; CHECK: // %bb.0:1199; CHECK-NEXT: ptrue p0.d, vl321200; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1201; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]1202; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z1.d1203; CHECK-NEXT: st1d { z0.d }, p0, [x0]1204; CHECK-NEXT: ret1205 %op1 = load <32 x i64>, ptr %a1206 %op2 = load <32 x i64>, ptr %b1207 %res = shl <32 x i64> %op1, %op21208 store <32 x i64> %res, ptr %a1209 ret void1210}1211 1212attributes #0 = { "target-features"="+sve" }1213