351 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming -enable-subreg-liveness -verify-machineinstrs < %s | FileCheck %s3 4;5; S/UQRSHR x26;7 8define <vscale x 8 x i16> @multi_vector_sat_shift_narrow_x2_s16(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2) {9; CHECK-LABEL: multi_vector_sat_shift_narrow_x2_s16:10; CHECK: // %bb.0:11; CHECK-NEXT: mov z3.d, z2.d12; CHECK-NEXT: mov z2.d, z1.d13; CHECK-NEXT: sqrshr z0.h, { z2.s, z3.s }, #1614; CHECK-NEXT: ret15 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshr.x2.nxv8i16(<vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, i32 16)16 ret <vscale x 8 x i16> %res17}18 19define { <vscale x 8 x i16>, <vscale x 8 x i16> } @multi_vector_sat_shift_narrow_x2_s16_tuple(i64 %stride, ptr %ptr) {20; CHECK-LABEL: multi_vector_sat_shift_narrow_x2_s16_tuple:21; CHECK: // %bb.0: // %entry22; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill23; CHECK-NEXT: addvl sp, sp, #-324; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill25; CHECK-NEXT: str z11, [sp, #1, mul vl] // 16-byte Folded Spill26; CHECK-NEXT: str z10, [sp, #2, mul vl] // 16-byte Folded Spill27; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG28; CHECK-NEXT: .cfi_offset w29, -1629; CHECK-NEXT: .cfi_escape 0x10, 0x4a, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d10 @ cfa - 8 * VG - 1630; CHECK-NEXT: .cfi_escape 0x10, 0x4b, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d11 @ cfa - 16 * VG - 1631; CHECK-NEXT: ptrue pn8.b32; CHECK-NEXT: add x8, x1, x033; CHECK-NEXT: ld1w { z2.s, z10.s }, pn8/z, [x1]34; CHECK-NEXT: ld1w { z3.s, z11.s }, pn8/z, [x8]35; CHECK-NEXT: sqrshr z0.h, { z2.s, z3.s }, #1636; CHECK-NEXT: sqrshr z1.h, { z10.s, z11.s }, #1637; CHECK-NEXT: ldr z11, [sp, #1, mul vl] // 16-byte Folded Reload38; CHECK-NEXT: ldr z10, [sp, #2, mul vl] // 16-byte Folded Reload39; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload40; CHECK-NEXT: addvl sp, sp, #341; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload42; CHECK-NEXT: ret43entry:44 %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()45 %1 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount") %0, ptr %ptr)46 %2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %1, 047 %3 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %1, 148 %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride49 %4 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount") %0, ptr %arrayidx2)50 %5 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %4, 051 %6 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %4, 152 %res1 = call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshr.x2.nxv8i16(<vscale x 4 x i32> %2, <vscale x 4 x i32> %5, i32 16)53 %res2 = call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshr.x2.nxv8i16(<vscale x 4 x i32> %3, <vscale x 4 x i32> %6, i32 16)54 %ins1 = insertvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } poison, <vscale x 8 x i16> %res1, 055 %ins2 = insertvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %ins1, <vscale x 8 x i16> %res2, 156 ret { <vscale x 8 x i16>, <vscale x 8 x i16> } %ins257}58 59define <vscale x 8 x i16> @multi_vector_sat_shift_narrow_x2_u16(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2) {60; CHECK-LABEL: multi_vector_sat_shift_narrow_x2_u16:61; CHECK: // %bb.0:62; CHECK-NEXT: mov z3.d, z2.d63; CHECK-NEXT: mov z2.d, z1.d64; CHECK-NEXT: uqrshr z0.h, { z2.s, z3.s }, #1665; CHECK-NEXT: ret66 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.uqrshr.x2.nxv8i16(<vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, i32 16)67 ret <vscale x 8 x i16> %res68}69 70;71; S/UQRSHR x472;73 74define <vscale x 16 x i8> @multi_vector_sat_shift_narrow_x4_s8(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4) {75; CHECK-LABEL: multi_vector_sat_shift_narrow_x4_s8:76; CHECK: // %bb.0:77; CHECK-NEXT: mov z7.d, z4.d78; CHECK-NEXT: mov z6.d, z3.d79; CHECK-NEXT: mov z5.d, z2.d80; CHECK-NEXT: mov z4.d, z1.d81; CHECK-NEXT: sqrshr z0.b, { z4.s - z7.s }, #3282; CHECK-NEXT: ret83 %res = call <vscale x 16 x i8> @llvm.aarch64.sve.sqrshr.x4.nxv16i8(<vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4, i32 32)84 ret <vscale x 16 x i8> %res85}86 87define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @multi_vector_sat_shift_narrow_x4_s8_tuple(i64 %stride, ptr %ptr) {88; CHECK-LABEL: multi_vector_sat_shift_narrow_x4_s8_tuple:89; CHECK: // %bb.0: // %entry90; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill91; CHECK-NEXT: addvl sp, sp, #-992; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill93; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill94; CHECK-NEXT: str z22, [sp, #2, mul vl] // 16-byte Folded Spill95; CHECK-NEXT: str z21, [sp, #3, mul vl] // 16-byte Folded Spill96; CHECK-NEXT: str z20, [sp, #4, mul vl] // 16-byte Folded Spill97; CHECK-NEXT: str z19, [sp, #5, mul vl] // 16-byte Folded Spill98; CHECK-NEXT: str z18, [sp, #6, mul vl] // 16-byte Folded Spill99; CHECK-NEXT: str z17, [sp, #7, mul vl] // 16-byte Folded Spill100; CHECK-NEXT: str z16, [sp, #8, mul vl] // 16-byte Folded Spill101; CHECK-NEXT: .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0xc8, 0x00, 0x1e, 0x22 // sp + 16 + 72 * VG102; CHECK-NEXT: .cfi_offset w29, -16103; CHECK-NEXT: lsl x8, x0, #1104; CHECK-NEXT: add x9, x1, x0105; CHECK-NEXT: ptrue pn8.b106; CHECK-NEXT: ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x1]107; CHECK-NEXT: ld1w { z17.s, z21.s, z25.s, z29.s }, pn8/z, [x9]108; CHECK-NEXT: add x10, x1, x8109; CHECK-NEXT: add x8, x9, x8110; CHECK-NEXT: ld1w { z18.s, z22.s, z26.s, z30.s }, pn8/z, [x10]111; CHECK-NEXT: ld1w { z19.s, z23.s, z27.s, z31.s }, pn8/z, [x8]112; CHECK-NEXT: sqrshr z0.b, { z16.s - z19.s }, #32113; CHECK-NEXT: sqrshr z1.b, { z20.s - z23.s }, #32114; CHECK-NEXT: sqrshr z2.b, { z24.s - z27.s }, #32115; CHECK-NEXT: sqrshr z3.b, { z28.s - z31.s }, #32116; CHECK-NEXT: ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload117; CHECK-NEXT: ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload118; CHECK-NEXT: ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload119; CHECK-NEXT: ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload120; CHECK-NEXT: ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload121; CHECK-NEXT: ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload122; CHECK-NEXT: ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload123; CHECK-NEXT: ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload124; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload125; CHECK-NEXT: addvl sp, sp, #9126; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload127; CHECK-NEXT: ret128entry:129 %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()130 %1 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") %0, ptr %ptr)131 %2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %1, 0132 %3 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %1, 1133 %4 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %1, 2134 %5 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %1, 3135 %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride136 %6 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") %0, ptr %arrayidx2)137 %7 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %6, 0138 %8 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %6, 1139 %9 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %6, 2140 %10 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %6, 3141 %mul3 = shl i64 %stride, 1142 %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3143 %11 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") %0, ptr %arrayidx4)144 %12 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %11, 0145 %13 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %11, 1146 %14 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %11, 2147 %15 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %11, 3148 %mul5 = mul i64 %stride, 3149 %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5150 %16 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") %0, ptr %arrayidx6)151 %17 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %16, 0152 %18 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %16, 1153 %19 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %16, 2154 %20 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %16, 3155 %res1 = call <vscale x 16 x i8> @llvm.aarch64.sve.sqrshr.x4.nxv16i8(<vscale x 4 x i32> %2, <vscale x 4 x i32> %7, <vscale x 4 x i32> %12, <vscale x 4 x i32> %17, i32 32)156 %res2 = call <vscale x 16 x i8> @llvm.aarch64.sve.sqrshr.x4.nxv16i8(<vscale x 4 x i32> %3, <vscale x 4 x i32> %8, <vscale x 4 x i32> %13, <vscale x 4 x i32> %18, i32 32)157 %res3 = call <vscale x 16 x i8> @llvm.aarch64.sve.sqrshr.x4.nxv16i8(<vscale x 4 x i32> %4, <vscale x 4 x i32> %9, <vscale x 4 x i32> %14, <vscale x 4 x i32> %19, i32 32)158 %res4 = call <vscale x 16 x i8> @llvm.aarch64.sve.sqrshr.x4.nxv16i8(<vscale x 4 x i32> %5, <vscale x 4 x i32> %10, <vscale x 4 x i32> %15, <vscale x 4 x i32> %20, i32 32)159 %ins1 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } poison, <vscale x 16 x i8> %res1, 0160 %ins2 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %ins1, <vscale x 16 x i8> %res2, 1161 %ins3 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %ins2, <vscale x 16 x i8> %res3, 2162 %ins4 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %ins3, <vscale x 16 x i8> %res4, 3163 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %ins4164}165 166define <vscale x 8 x i16> @multi_vector_sat_shift_narrow_x4_s16(<vscale x 2 x i64> %unused, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4) {167; CHECK-LABEL: multi_vector_sat_shift_narrow_x4_s16:168; CHECK: // %bb.0:169; CHECK-NEXT: mov z7.d, z4.d170; CHECK-NEXT: mov z6.d, z3.d171; CHECK-NEXT: mov z5.d, z2.d172; CHECK-NEXT: mov z4.d, z1.d173; CHECK-NEXT: sqrshr z0.h, { z4.d - z7.d }, #64174; CHECK-NEXT: ret175 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshr.x4.nxv8i16(<vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4, i32 64)176 ret <vscale x 8 x i16> %res177}178 179define <vscale x 16 x i8> @multi_vector_sat_shift_narrow_x4_u8(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4) {180; CHECK-LABEL: multi_vector_sat_shift_narrow_x4_u8:181; CHECK: // %bb.0:182; CHECK-NEXT: mov z7.d, z4.d183; CHECK-NEXT: mov z6.d, z3.d184; CHECK-NEXT: mov z5.d, z2.d185; CHECK-NEXT: mov z4.d, z1.d186; CHECK-NEXT: uqrshr z0.b, { z4.s - z7.s }, #32187; CHECK-NEXT: ret188 %res = call <vscale x 16 x i8> @llvm.aarch64.sve.uqrshr.x4.nxv16i8(<vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4, i32 32)189 ret <vscale x 16 x i8> %res190}191 192define <vscale x 8 x i16> @multi_vector_sat_shift_narrow_x4_u16(<vscale x 2 x i64> %unused, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4) {193; CHECK-LABEL: multi_vector_sat_shift_narrow_x4_u16:194; CHECK: // %bb.0:195; CHECK-NEXT: mov z7.d, z4.d196; CHECK-NEXT: mov z6.d, z3.d197; CHECK-NEXT: mov z5.d, z2.d198; CHECK-NEXT: mov z4.d, z1.d199; CHECK-NEXT: uqrshr z0.h, { z4.d - z7.d }, #64200; CHECK-NEXT: ret201 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.uqrshr.x4.nxv8i16(<vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4, i32 64)202 ret <vscale x 8 x i16> %res203}204 205; S/UQRSHRN x4206 207define <vscale x 16 x i8> @multi_vector_sat_shift_narrow_interleave_x4_s8(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4) {208; CHECK-LABEL: multi_vector_sat_shift_narrow_interleave_x4_s8:209; CHECK: // %bb.0:210; CHECK-NEXT: mov z7.d, z4.d211; CHECK-NEXT: mov z6.d, z3.d212; CHECK-NEXT: mov z5.d, z2.d213; CHECK-NEXT: mov z4.d, z1.d214; CHECK-NEXT: sqrshrn z0.b, { z4.s - z7.s }, #32215; CHECK-NEXT: ret216 %res = call <vscale x 16 x i8> @llvm.aarch64.sve.sqrshrn.x4.nxv16i8(<vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4, i32 32)217 ret <vscale x 16 x i8> %res218}219 220define <vscale x 8 x i16> @multi_vector_sat_shift_narrow_interleave_x4_s16(<vscale x 2 x i64> %unused, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4) {221; CHECK-LABEL: multi_vector_sat_shift_narrow_interleave_x4_s16:222; CHECK: // %bb.0:223; CHECK-NEXT: mov z7.d, z4.d224; CHECK-NEXT: mov z6.d, z3.d225; CHECK-NEXT: mov z5.d, z2.d226; CHECK-NEXT: mov z4.d, z1.d227; CHECK-NEXT: sqrshrn z0.h, { z4.d - z7.d }, #64228; CHECK-NEXT: ret229 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrn.x4.nxv8i16(<vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4, i32 64)230 ret <vscale x 8 x i16> %res231}232 233define <vscale x 16 x i8> @multi_vector_sat_shift_narrow_interleave_x4_u8(<vscale x 2 x i64> %unused, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4) {234; CHECK-LABEL: multi_vector_sat_shift_narrow_interleave_x4_u8:235; CHECK: // %bb.0:236; CHECK-NEXT: mov z7.d, z4.d237; CHECK-NEXT: mov z6.d, z3.d238; CHECK-NEXT: mov z5.d, z2.d239; CHECK-NEXT: mov z4.d, z1.d240; CHECK-NEXT: uqrshrn z0.b, { z4.s - z7.s }, #32241; CHECK-NEXT: ret242 %res = call <vscale x 16 x i8> @llvm.aarch64.sve.uqrshrn.x4.nxv16i8(<vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4, i32 32)243 ret <vscale x 16 x i8> %res244}245 246define <vscale x 8 x i16> @multi_vector_sat_shift_narrow_interleave_x4_u16(<vscale x 2 x i64> %unused, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4) {247; CHECK-LABEL: multi_vector_sat_shift_narrow_interleave_x4_u16:248; CHECK: // %bb.0:249; CHECK-NEXT: mov z7.d, z4.d250; CHECK-NEXT: mov z6.d, z3.d251; CHECK-NEXT: mov z5.d, z2.d252; CHECK-NEXT: mov z4.d, z1.d253; CHECK-NEXT: uqrshrn z0.h, { z4.d - z7.d }, #64254; CHECK-NEXT: ret255 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.uqrshrn.x4.nxv8i16(<vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4, i32 64)256 ret <vscale x 8 x i16> %res257}258 259; SQRSHRU x2260 261define <vscale x 8 x i16> @multi_vector_sat_shift_unsigned_narrow_x2_u16(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2) {262; CHECK-LABEL: multi_vector_sat_shift_unsigned_narrow_x2_u16:263; CHECK: // %bb.0:264; CHECK-NEXT: mov z3.d, z2.d265; CHECK-NEXT: mov z2.d, z1.d266; CHECK-NEXT: sqrshru z0.h, { z2.s, z3.s }, #16267; CHECK-NEXT: ret268 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshru.x2.nxv8i16(<vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, i32 16)269 ret <vscale x 8 x i16> %res270}271 272; SQRSHRU x4273 274define <vscale x 16 x i8> @multi_vector_sat_shift_unsigned_narrow_x4_u8(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4) {275; CHECK-LABEL: multi_vector_sat_shift_unsigned_narrow_x4_u8:276; CHECK: // %bb.0:277; CHECK-NEXT: mov z7.d, z4.d278; CHECK-NEXT: mov z6.d, z3.d279; CHECK-NEXT: mov z5.d, z2.d280; CHECK-NEXT: mov z4.d, z1.d281; CHECK-NEXT: sqrshru z0.b, { z4.s - z7.s }, #32282; CHECK-NEXT: ret283 %res = call <vscale x 16 x i8> @llvm.aarch64.sve.sqrshru.x4.nxv16i8(<vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4, i32 32)284 ret <vscale x 16 x i8> %res285}286 287define <vscale x 8 x i16> @multi_vector_sat_shift_unsigned_narrow_x4_u16(<vscale x 2 x i64> %unused, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4) {288; CHECK-LABEL: multi_vector_sat_shift_unsigned_narrow_x4_u16:289; CHECK: // %bb.0:290; CHECK-NEXT: mov z7.d, z4.d291; CHECK-NEXT: mov z6.d, z3.d292; CHECK-NEXT: mov z5.d, z2.d293; CHECK-NEXT: mov z4.d, z1.d294; CHECK-NEXT: sqrshru z0.h, { z4.d - z7.d }, #64295; CHECK-NEXT: ret296 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshru.x4.nxv8i16(<vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4, i32 64)297 ret <vscale x 8 x i16> %res298}299 300; SQRSHRUN x4301 302define <vscale x 16 x i8> @multi_vector_sat_shift_unsigned_narrow_interleave_x4_u8(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4) {303; CHECK-LABEL: multi_vector_sat_shift_unsigned_narrow_interleave_x4_u8:304; CHECK: // %bb.0:305; CHECK-NEXT: mov z7.d, z4.d306; CHECK-NEXT: mov z6.d, z3.d307; CHECK-NEXT: mov z5.d, z2.d308; CHECK-NEXT: mov z4.d, z1.d309; CHECK-NEXT: sqrshrun z0.b, { z4.s - z7.s }, #32310; CHECK-NEXT: ret311 %res = call <vscale x 16 x i8> @llvm.aarch64.sve.sqrshrun.x4.nxv16i8(<vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4, i32 32)312 ret <vscale x 16 x i8> %res313}314 315define <vscale x 8 x i16> @multi_vector_sat_shift_unsigned_narrow_interleave_x4_u16(<vscale x 2 x i64> %unused, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4) {316; CHECK-LABEL: multi_vector_sat_shift_unsigned_narrow_interleave_x4_u16:317; CHECK: // %bb.0:318; CHECK-NEXT: mov z7.d, z4.d319; CHECK-NEXT: mov z6.d, z3.d320; CHECK-NEXT: mov z5.d, z2.d321; CHECK-NEXT: mov z4.d, z1.d322; CHECK-NEXT: sqrshrun z0.h, { z4.d - z7.d }, #64323; CHECK-NEXT: ret324 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrun.x4.nxv8i16(<vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4, i32 64)325 ret <vscale x 8 x i16> %res326}327 328declare <vscale x 8 x i16> @llvm.aarch64.sve.sqrshr.x2.nxv8i16(<vscale x 4 x i32>, <vscale x 4 x i32>, i32)329 330declare <vscale x 16 x i8> @llvm.aarch64.sve.sqrshr.x4.nxv16i8(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, i32)331declare <vscale x 8 x i16> @llvm.aarch64.sve.sqrshr.x4.nxv8i16(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, i32)332 333declare <vscale x 8 x i16> @llvm.aarch64.sve.uqrshr.x2.nxv8i16(<vscale x 4 x i32>, <vscale x 4 x i32>, i32)334 335declare <vscale x 16 x i8> @llvm.aarch64.sve.uqrshr.x4.nxv16i8(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, i32)336declare <vscale x 8 x i16> @llvm.aarch64.sve.uqrshr.x4.nxv8i16(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, i32)337 338declare <vscale x 16 x i8> @llvm.aarch64.sve.sqrshrn.x4.nxv16i8(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, i32)339declare <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrn.x4.nxv8i16(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, i32)340 341declare <vscale x 16 x i8> @llvm.aarch64.sve.uqrshrn.x4.nxv16i8(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, i32)342declare <vscale x 8 x i16> @llvm.aarch64.sve.uqrshrn.x4.nxv8i16(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, i32)343 344declare <vscale x 8 x i16> @llvm.aarch64.sve.sqrshru.x2.nxv8i16(<vscale x 4 x i32>, <vscale x 4 x i32>, i32)345 346declare <vscale x 16 x i8> @llvm.aarch64.sve.sqrshru.x4.nxv16i8(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, i32)347declare <vscale x 8 x i16> @llvm.aarch64.sve.sqrshru.x4.nxv8i16(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, i32)348 349declare <vscale x 16 x i8> @llvm.aarch64.sve.sqrshrun.x4.nxv16i8(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, i32)350declare <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrun.x4.nxv8i16(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, i32)351