870 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mattr=+sve | FileCheck %s -check-prefixes=CHECK,SVE3; RUN: llc < %s -mattr=+sve,+sme2 | FileCheck %s -check-prefixes=CHECK,SVE4; RUN: llc < %s -mattr=+sme2 -force-streaming | FileCheck %s -check-prefixes=CHECK,SME2,SME2-ALL5; RUN: llc < %s -mattr=+sme2 -force-streaming -aarch64-sve-vector-bits-min=256 | FileCheck %s -check-prefixes=CHECK,SME2,SME2-2566 7target triple = "aarch64-unknown-linux-gnu"8 9define {<vscale x 2 x half>, <vscale x 2 x half>} @vector_deinterleave_nxv2f16_nxv4f16(<vscale x 4 x half> %vec) {10; SVE-LABEL: vector_deinterleave_nxv2f16_nxv4f16:11; SVE: // %bb.0:12; SVE-NEXT: uzp1 z1.s, z0.s, z0.s13; SVE-NEXT: uzp2 z2.s, z0.s, z0.s14; SVE-NEXT: uunpklo z0.d, z1.s15; SVE-NEXT: uunpklo z1.d, z2.s16; SVE-NEXT: ret17;18; SME2-LABEL: vector_deinterleave_nxv2f16_nxv4f16:19; SME2: // %bb.0:20; SME2-NEXT: uunpkhi z1.d, z0.s21; SME2-NEXT: uunpklo z0.d, z0.s22; SME2-NEXT: uzp { z0.d, z1.d }, z0.d, z1.d23; SME2-NEXT: ret24 %retval = call {<vscale x 2 x half>, <vscale x 2 x half>} @llvm.vector.deinterleave2.nxv4f16(<vscale x 4 x half> %vec)25 ret {<vscale x 2 x half>, <vscale x 2 x half>} %retval26}27 28define {<vscale x 4 x half>, <vscale x 4 x half>} @vector_deinterleave_nxv4f16_nxv8f16(<vscale x 8 x half> %vec) {29; SVE-LABEL: vector_deinterleave_nxv4f16_nxv8f16:30; SVE: // %bb.0:31; SVE-NEXT: uzp1 z1.h, z0.h, z0.h32; SVE-NEXT: uzp2 z2.h, z0.h, z0.h33; SVE-NEXT: uunpklo z0.s, z1.h34; SVE-NEXT: uunpklo z1.s, z2.h35; SVE-NEXT: ret36;37; SME2-LABEL: vector_deinterleave_nxv4f16_nxv8f16:38; SME2: // %bb.0:39; SME2-NEXT: uunpkhi z1.s, z0.h40; SME2-NEXT: uunpklo z0.s, z0.h41; SME2-NEXT: uzp { z0.s, z1.s }, z0.s, z1.s42; SME2-NEXT: ret43 %retval = call {<vscale x 4 x half>, <vscale x 4 x half>} @llvm.vector.deinterleave2.nxv8f16(<vscale x 8 x half> %vec)44 ret {<vscale x 4 x half>, <vscale x 4 x half>} %retval45}46 47define {<vscale x 8 x half>, <vscale x 8 x half>} @vector_deinterleave_nxv8f16_nxv16f16(<vscale x 16 x half> %vec) {48; SVE-LABEL: vector_deinterleave_nxv8f16_nxv16f16:49; SVE: // %bb.0:50; SVE-NEXT: uzp1 z2.h, z0.h, z1.h51; SVE-NEXT: uzp2 z1.h, z0.h, z1.h52; SVE-NEXT: mov z0.d, z2.d53; SVE-NEXT: ret54;55; SME2-LABEL: vector_deinterleave_nxv8f16_nxv16f16:56; SME2: // %bb.0:57; SME2-NEXT: uzp { z0.h, z1.h }, z0.h, z1.h58; SME2-NEXT: ret59 %retval = call {<vscale x 8 x half>, <vscale x 8 x half>} @llvm.vector.deinterleave2.nxv16f16(<vscale x 16 x half> %vec)60 ret {<vscale x 8 x half>, <vscale x 8 x half>} %retval61}62 63define {<vscale x 2 x float>, <vscale x 2 x float>} @vector_deinterleave_nxv2f32_nxv4f32(<vscale x 4 x float> %vec) {64; SVE-LABEL: vector_deinterleave_nxv2f32_nxv4f32:65; SVE: // %bb.0:66; SVE-NEXT: uzp1 z1.s, z0.s, z0.s67; SVE-NEXT: uzp2 z2.s, z0.s, z0.s68; SVE-NEXT: uunpklo z0.d, z1.s69; SVE-NEXT: uunpklo z1.d, z2.s70; SVE-NEXT: ret71;72; SME2-LABEL: vector_deinterleave_nxv2f32_nxv4f32:73; SME2: // %bb.0:74; SME2-NEXT: uunpkhi z1.d, z0.s75; SME2-NEXT: uunpklo z0.d, z0.s76; SME2-NEXT: uzp { z0.d, z1.d }, z0.d, z1.d77; SME2-NEXT: ret78 %retval = call {<vscale x 2 x float>, <vscale x 2 x float>} @llvm.vector.deinterleave2.nxv4f32(<vscale x 4 x float> %vec)79 ret {<vscale x 2 x float>, <vscale x 2 x float>} %retval80}81 82define {<vscale x 4 x float>, <vscale x 4 x float>} @vector_deinterleave_nxv4f32_nxv8f32(<vscale x 8 x float> %vec) {83; SVE-LABEL: vector_deinterleave_nxv4f32_nxv8f32:84; SVE: // %bb.0:85; SVE-NEXT: uzp1 z2.s, z0.s, z1.s86; SVE-NEXT: uzp2 z1.s, z0.s, z1.s87; SVE-NEXT: mov z0.d, z2.d88; SVE-NEXT: ret89;90; SME2-LABEL: vector_deinterleave_nxv4f32_nxv8f32:91; SME2: // %bb.0:92; SME2-NEXT: uzp { z0.s, z1.s }, z0.s, z1.s93; SME2-NEXT: ret94 %retval = call {<vscale x 4 x float>, <vscale x 4 x float>} @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %vec)95 ret {<vscale x 4 x float>, <vscale x 4 x float>} %retval96}97 98define {<vscale x 2 x double>, <vscale x 2 x double>} @vector_deinterleave_nxv2f64_nxv4f64(<vscale x 4 x double> %vec) {99; SVE-LABEL: vector_deinterleave_nxv2f64_nxv4f64:100; SVE: // %bb.0:101; SVE-NEXT: uzp1 z2.d, z0.d, z1.d102; SVE-NEXT: uzp2 z1.d, z0.d, z1.d103; SVE-NEXT: mov z0.d, z2.d104; SVE-NEXT: ret105;106; SME2-LABEL: vector_deinterleave_nxv2f64_nxv4f64:107; SME2: // %bb.0:108; SME2-NEXT: uzp { z0.d, z1.d }, z0.d, z1.d109; SME2-NEXT: ret110 %retval = call {<vscale x 2 x double>, <vscale x 2 x double>} @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %vec)111 ret {<vscale x 2 x double>, <vscale x 2 x double>} %retval112}113 114define {<vscale x 2 x bfloat>, <vscale x 2 x bfloat>} @vector_deinterleave_nxv2bf16_nxv4bf16(<vscale x 4 x bfloat> %vec) {115; SVE-LABEL: vector_deinterleave_nxv2bf16_nxv4bf16:116; SVE: // %bb.0:117; SVE-NEXT: uzp1 z1.s, z0.s, z0.s118; SVE-NEXT: uzp2 z2.s, z0.s, z0.s119; SVE-NEXT: uunpklo z0.d, z1.s120; SVE-NEXT: uunpklo z1.d, z2.s121; SVE-NEXT: ret122;123; SME2-LABEL: vector_deinterleave_nxv2bf16_nxv4bf16:124; SME2: // %bb.0:125; SME2-NEXT: uunpkhi z1.d, z0.s126; SME2-NEXT: uunpklo z0.d, z0.s127; SME2-NEXT: uzp { z0.d, z1.d }, z0.d, z1.d128; SME2-NEXT: ret129 %retval = call {<vscale x 2 x bfloat>, <vscale x 2 x bfloat>} @llvm.vector.deinterleave2.nxv4bf16(<vscale x 4 x bfloat> %vec)130 ret {<vscale x 2 x bfloat>, <vscale x 2 x bfloat>} %retval131}132 133define {<vscale x 4 x bfloat>, <vscale x 4 x bfloat>} @vector_deinterleave_nxv4bf16_nxv8bf16(<vscale x 8 x bfloat> %vec) {134; SVE-LABEL: vector_deinterleave_nxv4bf16_nxv8bf16:135; SVE: // %bb.0:136; SVE-NEXT: uzp1 z1.h, z0.h, z0.h137; SVE-NEXT: uzp2 z2.h, z0.h, z0.h138; SVE-NEXT: uunpklo z0.s, z1.h139; SVE-NEXT: uunpklo z1.s, z2.h140; SVE-NEXT: ret141;142; SME2-LABEL: vector_deinterleave_nxv4bf16_nxv8bf16:143; SME2: // %bb.0:144; SME2-NEXT: uunpkhi z1.s, z0.h145; SME2-NEXT: uunpklo z0.s, z0.h146; SME2-NEXT: uzp { z0.s, z1.s }, z0.s, z1.s147; SME2-NEXT: ret148 %retval = call {<vscale x 4 x bfloat>, <vscale x 4 x bfloat>} @llvm.vector.deinterleave2.nxv8bf16(<vscale x 8 x bfloat> %vec)149 ret {<vscale x 4 x bfloat>, <vscale x 4 x bfloat>} %retval150}151 152define {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @vector_deinterleave_nxv8bf16_nxv16bf16(<vscale x 16 x bfloat> %vec) {153; SVE-LABEL: vector_deinterleave_nxv8bf16_nxv16bf16:154; SVE: // %bb.0:155; SVE-NEXT: uzp1 z2.h, z0.h, z1.h156; SVE-NEXT: uzp2 z1.h, z0.h, z1.h157; SVE-NEXT: mov z0.d, z2.d158; SVE-NEXT: ret159;160; SME2-LABEL: vector_deinterleave_nxv8bf16_nxv16bf16:161; SME2: // %bb.0:162; SME2-NEXT: uzp { z0.h, z1.h }, z0.h, z1.h163; SME2-NEXT: ret164 %retval = call {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.vector.deinterleave2.nxv16bf16(<vscale x 16 x bfloat> %vec)165 ret {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} %retval166}167 168; Integers169 170define {<vscale x 16 x i8>, <vscale x 16 x i8>} @vector_deinterleave_nxv16i8_nxv32i8(<vscale x 32 x i8> %vec) {171; SVE-LABEL: vector_deinterleave_nxv16i8_nxv32i8:172; SVE: // %bb.0:173; SVE-NEXT: uzp1 z2.b, z0.b, z1.b174; SVE-NEXT: uzp2 z1.b, z0.b, z1.b175; SVE-NEXT: mov z0.d, z2.d176; SVE-NEXT: ret177;178; SME2-LABEL: vector_deinterleave_nxv16i8_nxv32i8:179; SME2: // %bb.0:180; SME2-NEXT: uzp { z0.b, z1.b }, z0.b, z1.b181; SME2-NEXT: ret182 %retval = call {<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> %vec)183 ret {<vscale x 16 x i8>, <vscale x 16 x i8>} %retval184}185 186define {<vscale x 8 x i16>, <vscale x 8 x i16>} @vector_deinterleave_nxv8i16_nxv16i16(<vscale x 16 x i16> %vec) {187; SVE-LABEL: vector_deinterleave_nxv8i16_nxv16i16:188; SVE: // %bb.0:189; SVE-NEXT: uzp1 z2.h, z0.h, z1.h190; SVE-NEXT: uzp2 z1.h, z0.h, z1.h191; SVE-NEXT: mov z0.d, z2.d192; SVE-NEXT: ret193;194; SME2-LABEL: vector_deinterleave_nxv8i16_nxv16i16:195; SME2: // %bb.0:196; SME2-NEXT: uzp { z0.h, z1.h }, z0.h, z1.h197; SME2-NEXT: ret198 %retval = call {<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.vector.deinterleave2.nxv16i16(<vscale x 16 x i16> %vec)199 ret {<vscale x 8 x i16>, <vscale x 8 x i16>} %retval200}201 202define {<vscale x 4 x i32>, <vscale x 4 x i32>} @vector_deinterleave_nxv4i32_nxvv8i32(<vscale x 8 x i32> %vec) {203; SVE-LABEL: vector_deinterleave_nxv4i32_nxvv8i32:204; SVE: // %bb.0:205; SVE-NEXT: uzp1 z2.s, z0.s, z1.s206; SVE-NEXT: uzp2 z1.s, z0.s, z1.s207; SVE-NEXT: mov z0.d, z2.d208; SVE-NEXT: ret209;210; SME2-LABEL: vector_deinterleave_nxv4i32_nxvv8i32:211; SME2: // %bb.0:212; SME2-NEXT: uzp { z0.s, z1.s }, z0.s, z1.s213; SME2-NEXT: ret214 %retval = call {<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %vec)215 ret {<vscale x 4 x i32>, <vscale x 4 x i32>} %retval216}217 218define {<vscale x 2 x i64>, <vscale x 2 x i64>} @vector_deinterleave_nxv2i64_nxv4i64(<vscale x 4 x i64> %vec) {219; SVE-LABEL: vector_deinterleave_nxv2i64_nxv4i64:220; SVE: // %bb.0:221; SVE-NEXT: uzp1 z2.d, z0.d, z1.d222; SVE-NEXT: uzp2 z1.d, z0.d, z1.d223; SVE-NEXT: mov z0.d, z2.d224; SVE-NEXT: ret225;226; SME2-LABEL: vector_deinterleave_nxv2i64_nxv4i64:227; SME2: // %bb.0:228; SME2-NEXT: uzp { z0.d, z1.d }, z0.d, z1.d229; SME2-NEXT: ret230 %retval = call {<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> %vec)231 ret {<vscale x 2 x i64>, <vscale x 2 x i64>} %retval232}233 234define {<vscale x 2 x half>, <vscale x 2 x half>, <vscale x 2 x half>} @vector_deinterleave_nxv2f16_nxv6f16(<vscale x 6 x half> %vec) {235; CHECK-LABEL: vector_deinterleave_nxv2f16_nxv6f16:236; CHECK: // %bb.0:237; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill238; CHECK-NEXT: addvl sp, sp, #-3239; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG240; CHECK-NEXT: .cfi_offset w29, -16241; CHECK-NEXT: uunpkhi z1.s, z0.h242; CHECK-NEXT: uunpklo z0.s, z0.h243; CHECK-NEXT: ptrue p0.d244; CHECK-NEXT: uunpklo z1.d, z1.s245; CHECK-NEXT: uunpkhi z2.d, z0.s246; CHECK-NEXT: uunpklo z0.d, z0.s247; CHECK-NEXT: str z1, [sp, #2, mul vl]248; CHECK-NEXT: str z2, [sp, #1, mul vl]249; CHECK-NEXT: str z0, [sp]250; CHECK-NEXT: ld3d { z0.d - z2.d }, p0/z, [sp]251; CHECK-NEXT: addvl sp, sp, #3252; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload253; CHECK-NEXT: ret254 %retval = call {<vscale x 2 x half>, <vscale x 2 x half>, <vscale x 2 x half>} @llvm.vector.deinterleave3.nxv6f16(<vscale x 6 x half> %vec)255 ret {<vscale x 2 x half>, <vscale x 2 x half>, <vscale x 2 x half>} %retval256}257 258define {<vscale x 4 x half>, <vscale x 4 x half>, <vscale x 4 x half>} @vector_deinterleave_nxv4f16_nxv12f16(<vscale x 12 x half> %vec) {259; CHECK-LABEL: vector_deinterleave_nxv4f16_nxv12f16:260; CHECK: // %bb.0:261; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill262; CHECK-NEXT: addvl sp, sp, #-3263; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG264; CHECK-NEXT: .cfi_offset w29, -16265; CHECK-NEXT: uunpklo z1.s, z1.h266; CHECK-NEXT: uunpkhi z2.s, z0.h267; CHECK-NEXT: uunpklo z0.s, z0.h268; CHECK-NEXT: ptrue p0.s269; CHECK-NEXT: str z1, [sp, #2, mul vl]270; CHECK-NEXT: str z2, [sp, #1, mul vl]271; CHECK-NEXT: str z0, [sp]272; CHECK-NEXT: ld3w { z0.s - z2.s }, p0/z, [sp]273; CHECK-NEXT: addvl sp, sp, #3274; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload275; CHECK-NEXT: ret276 %retval = call {<vscale x 4 x half>, <vscale x 4 x half>, <vscale x 4 x half>} @llvm.vector.deinterleave3.nxv12f16(<vscale x 12 x half> %vec)277 ret {<vscale x 4 x half>, <vscale x 4 x half>, <vscale x 4 x half>} %retval278}279 280define {<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>} @vector_deinterleave_nxv8f16_nxv24f16(<vscale x 24 x half> %vec) {281; CHECK-LABEL: vector_deinterleave_nxv8f16_nxv24f16:282; CHECK: // %bb.0:283; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill284; CHECK-NEXT: addvl sp, sp, #-3285; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG286; CHECK-NEXT: .cfi_offset w29, -16287; CHECK-NEXT: ptrue p0.h288; CHECK-NEXT: str z2, [sp, #2, mul vl]289; CHECK-NEXT: str z1, [sp, #1, mul vl]290; CHECK-NEXT: str z0, [sp]291; CHECK-NEXT: ld3h { z0.h - z2.h }, p0/z, [sp]292; CHECK-NEXT: addvl sp, sp, #3293; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload294; CHECK-NEXT: ret295 %retval = call {<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>} @llvm.vector.deinterleave3.nxv24f16(<vscale x 24 x half> %vec)296 ret {<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>} %retval297}298 299define {<vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>} @vector_deinterleave_nxv2f32_nxv6f32(<vscale x 6 x float> %vec) {300; CHECK-LABEL: vector_deinterleave_nxv2f32_nxv6f32:301; CHECK: // %bb.0:302; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill303; CHECK-NEXT: addvl sp, sp, #-3304; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG305; CHECK-NEXT: .cfi_offset w29, -16306; CHECK-NEXT: uunpklo z1.d, z1.s307; CHECK-NEXT: uunpkhi z2.d, z0.s308; CHECK-NEXT: uunpklo z0.d, z0.s309; CHECK-NEXT: ptrue p0.d310; CHECK-NEXT: str z1, [sp, #2, mul vl]311; CHECK-NEXT: str z2, [sp, #1, mul vl]312; CHECK-NEXT: str z0, [sp]313; CHECK-NEXT: ld3d { z0.d - z2.d }, p0/z, [sp]314; CHECK-NEXT: addvl sp, sp, #3315; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload316; CHECK-NEXT: ret317 %retval = call {<vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>} @llvm.vector.deinterleave3.nxv6f32(<vscale x 6 x float> %vec)318 ret {<vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>} %retval319}320 321define {<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>} @vector_deinterleave_nxv4f32_nxv12f32(<vscale x 12 x float> %vec) {322; CHECK-LABEL: vector_deinterleave_nxv4f32_nxv12f32:323; CHECK: // %bb.0:324; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill325; CHECK-NEXT: addvl sp, sp, #-3326; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG327; CHECK-NEXT: .cfi_offset w29, -16328; CHECK-NEXT: ptrue p0.s329; CHECK-NEXT: str z2, [sp, #2, mul vl]330; CHECK-NEXT: str z1, [sp, #1, mul vl]331; CHECK-NEXT: str z0, [sp]332; CHECK-NEXT: ld3w { z0.s - z2.s }, p0/z, [sp]333; CHECK-NEXT: addvl sp, sp, #3334; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload335; CHECK-NEXT: ret336 %retval = call {<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>} @llvm.vector.deinterleave3.nxv12f32(<vscale x 12 x float> %vec)337 ret {<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>} %retval338}339 340define {<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>} @vector_deinterleave_nxv2f64_nxv6f64(<vscale x 6 x double> %vec) {341; CHECK-LABEL: vector_deinterleave_nxv2f64_nxv6f64:342; CHECK: // %bb.0:343; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill344; CHECK-NEXT: addvl sp, sp, #-3345; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG346; CHECK-NEXT: .cfi_offset w29, -16347; CHECK-NEXT: ptrue p0.d348; CHECK-NEXT: str z2, [sp, #2, mul vl]349; CHECK-NEXT: str z1, [sp, #1, mul vl]350; CHECK-NEXT: str z0, [sp]351; CHECK-NEXT: ld3d { z0.d - z2.d }, p0/z, [sp]352; CHECK-NEXT: addvl sp, sp, #3353; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload354; CHECK-NEXT: ret355 %retval = call {<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>} @llvm.vector.deinterleave3.nxv6f64(<vscale x 6 x double> %vec)356 ret {<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>} %retval357}358 359define {<vscale x 2 x bfloat>, <vscale x 2 x bfloat>, <vscale x 2 x bfloat>} @vector_deinterleave_nxv2bf16_nxv6bf16(<vscale x 6 x bfloat> %vec) {360; CHECK-LABEL: vector_deinterleave_nxv2bf16_nxv6bf16:361; CHECK: // %bb.0:362; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill363; CHECK-NEXT: addvl sp, sp, #-3364; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG365; CHECK-NEXT: .cfi_offset w29, -16366; CHECK-NEXT: uunpkhi z1.s, z0.h367; CHECK-NEXT: uunpklo z0.s, z0.h368; CHECK-NEXT: ptrue p0.d369; CHECK-NEXT: uunpklo z1.d, z1.s370; CHECK-NEXT: uunpkhi z2.d, z0.s371; CHECK-NEXT: uunpklo z0.d, z0.s372; CHECK-NEXT: str z1, [sp, #2, mul vl]373; CHECK-NEXT: str z2, [sp, #1, mul vl]374; CHECK-NEXT: str z0, [sp]375; CHECK-NEXT: ld3d { z0.d - z2.d }, p0/z, [sp]376; CHECK-NEXT: addvl sp, sp, #3377; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload378; CHECK-NEXT: ret379 %retval = call {<vscale x 2 x bfloat>, <vscale x 2 x bfloat>, <vscale x 2 x bfloat>} @llvm.vector.deinterleave3.nxv6bf16(<vscale x 6 x bfloat> %vec)380 ret {<vscale x 2 x bfloat>, <vscale x 2 x bfloat>, <vscale x 2 x bfloat>} %retval381}382 383define {<vscale x 4 x bfloat>, <vscale x 4 x bfloat>, <vscale x 4 x bfloat>} @vector_deinterleave_nxv4bf16_nxv12bf16(<vscale x 12 x bfloat> %vec) {384; CHECK-LABEL: vector_deinterleave_nxv4bf16_nxv12bf16:385; CHECK: // %bb.0:386; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill387; CHECK-NEXT: addvl sp, sp, #-3388; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG389; CHECK-NEXT: .cfi_offset w29, -16390; CHECK-NEXT: uunpklo z1.s, z1.h391; CHECK-NEXT: uunpkhi z2.s, z0.h392; CHECK-NEXT: uunpklo z0.s, z0.h393; CHECK-NEXT: ptrue p0.s394; CHECK-NEXT: str z1, [sp, #2, mul vl]395; CHECK-NEXT: str z2, [sp, #1, mul vl]396; CHECK-NEXT: str z0, [sp]397; CHECK-NEXT: ld3w { z0.s - z2.s }, p0/z, [sp]398; CHECK-NEXT: addvl sp, sp, #3399; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload400; CHECK-NEXT: ret401 %retval = call {<vscale x 4 x bfloat>, <vscale x 4 x bfloat>, <vscale x 4 x bfloat>} @llvm.vector.deinterleave3.nxv12bf16(<vscale x 12 x bfloat> %vec)402 ret {<vscale x 4 x bfloat>, <vscale x 4 x bfloat>, <vscale x 4 x bfloat>} %retval403}404 405define {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @vector_deinterleave_nxv8bf16_nxv24bf16(<vscale x 24 x bfloat> %vec) {406; CHECK-LABEL: vector_deinterleave_nxv8bf16_nxv24bf16:407; CHECK: // %bb.0:408; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill409; CHECK-NEXT: addvl sp, sp, #-3410; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG411; CHECK-NEXT: .cfi_offset w29, -16412; CHECK-NEXT: ptrue p0.h413; CHECK-NEXT: str z2, [sp, #2, mul vl]414; CHECK-NEXT: str z1, [sp, #1, mul vl]415; CHECK-NEXT: str z0, [sp]416; CHECK-NEXT: ld3h { z0.h - z2.h }, p0/z, [sp]417; CHECK-NEXT: addvl sp, sp, #3418; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload419; CHECK-NEXT: ret420 %retval = call {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.vector.deinterleave3.nxv24bf16(<vscale x 24 x bfloat> %vec)421 ret {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>} %retval422}423 424; Integers425 426define {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @vector_deinterleave_nxv16i8_nxv48i8(<vscale x 48 x i8> %vec) {427; CHECK-LABEL: vector_deinterleave_nxv16i8_nxv48i8:428; CHECK: // %bb.0:429; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill430; CHECK-NEXT: addvl sp, sp, #-3431; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG432; CHECK-NEXT: .cfi_offset w29, -16433; CHECK-NEXT: ptrue p0.b434; CHECK-NEXT: str z2, [sp, #2, mul vl]435; CHECK-NEXT: str z1, [sp, #1, mul vl]436; CHECK-NEXT: str z0, [sp]437; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [sp]438; CHECK-NEXT: addvl sp, sp, #3439; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload440; CHECK-NEXT: ret441 %retval = call {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.vector.deinterleave3.nxv48i8(<vscale x 48 x i8> %vec)442 ret {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} %retval443}444 445define {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} @vector_deinterleave_nxv8i16_nxv24i16(<vscale x 24 x i16> %vec) {446; CHECK-LABEL: vector_deinterleave_nxv8i16_nxv24i16:447; CHECK: // %bb.0:448; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill449; CHECK-NEXT: addvl sp, sp, #-3450; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG451; CHECK-NEXT: .cfi_offset w29, -16452; CHECK-NEXT: ptrue p0.h453; CHECK-NEXT: str z2, [sp, #2, mul vl]454; CHECK-NEXT: str z1, [sp, #1, mul vl]455; CHECK-NEXT: str z0, [sp]456; CHECK-NEXT: ld3h { z0.h - z2.h }, p0/z, [sp]457; CHECK-NEXT: addvl sp, sp, #3458; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload459; CHECK-NEXT: ret460 %retval = call {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.vector.deinterleave3.nxv24i16(<vscale x 24 x i16> %vec)461 ret {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} %retval462}463 464define {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} @vector_deinterleave_nxv4i32_nxvv12i32(<vscale x 12 x i32> %vec) {465; CHECK-LABEL: vector_deinterleave_nxv4i32_nxvv12i32:466; CHECK: // %bb.0:467; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill468; CHECK-NEXT: addvl sp, sp, #-3469; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG470; CHECK-NEXT: .cfi_offset w29, -16471; CHECK-NEXT: ptrue p0.s472; CHECK-NEXT: str z2, [sp, #2, mul vl]473; CHECK-NEXT: str z1, [sp, #1, mul vl]474; CHECK-NEXT: str z0, [sp]475; CHECK-NEXT: ld3w { z0.s - z2.s }, p0/z, [sp]476; CHECK-NEXT: addvl sp, sp, #3477; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload478; CHECK-NEXT: ret479 %retval = call {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> %vec)480 ret {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} %retval481}482 483define {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @vector_deinterleave_nxv2i64_nxv6i64(<vscale x 6 x i64> %vec) {484; CHECK-LABEL: vector_deinterleave_nxv2i64_nxv6i64:485; CHECK: // %bb.0:486; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill487; CHECK-NEXT: addvl sp, sp, #-3488; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG489; CHECK-NEXT: .cfi_offset w29, -16490; CHECK-NEXT: ptrue p0.d491; CHECK-NEXT: str z2, [sp, #2, mul vl]492; CHECK-NEXT: str z1, [sp, #1, mul vl]493; CHECK-NEXT: str z0, [sp]494; CHECK-NEXT: ld3d { z0.d - z2.d }, p0/z, [sp]495; CHECK-NEXT: addvl sp, sp, #3496; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload497; CHECK-NEXT: ret498 %retval = call {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.vector.deinterleave3.nxv6i64(<vscale x 6 x i64> %vec)499 ret {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} %retval500}501 502define {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @vector_deinterleave_nxv16i8_nxv64i8(<vscale x 64 x i8> %vec) {503; SVE-LABEL: vector_deinterleave_nxv16i8_nxv64i8:504; SVE: // %bb.0:505; SVE-NEXT: uzp1 z4.b, z2.b, z3.b506; SVE-NEXT: uzp1 z5.b, z0.b, z1.b507; SVE-NEXT: uzp2 z3.b, z2.b, z3.b508; SVE-NEXT: uzp2 z6.b, z0.b, z1.b509; SVE-NEXT: uzp1 z0.b, z5.b, z4.b510; SVE-NEXT: uzp2 z2.b, z5.b, z4.b511; SVE-NEXT: uzp1 z1.b, z6.b, z3.b512; SVE-NEXT: uzp2 z3.b, z6.b, z3.b513; SVE-NEXT: ret514;515; SME2-LABEL: vector_deinterleave_nxv16i8_nxv64i8:516; SME2: // %bb.0:517; SME2-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3518; SME2-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3519; SME2-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3520; SME2-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3521; SME2-NEXT: uzp { z0.b - z3.b }, { z0.b - z3.b }522; SME2-NEXT: ret523 %retval = call {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.vector.deinterleave4.nxv64i8(<vscale x 64 x i8> %vec)524 ret {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} %retval525}526 527define {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} @vector_deinterleave_nxv8i16_nxv32i16(<vscale x 32 x i16> %vec) {528; SVE-LABEL: vector_deinterleave_nxv8i16_nxv32i16:529; SVE: // %bb.0:530; SVE-NEXT: uzp1 z4.h, z2.h, z3.h531; SVE-NEXT: uzp1 z5.h, z0.h, z1.h532; SVE-NEXT: uzp2 z3.h, z2.h, z3.h533; SVE-NEXT: uzp2 z6.h, z0.h, z1.h534; SVE-NEXT: uzp1 z0.h, z5.h, z4.h535; SVE-NEXT: uzp2 z2.h, z5.h, z4.h536; SVE-NEXT: uzp1 z1.h, z6.h, z3.h537; SVE-NEXT: uzp2 z3.h, z6.h, z3.h538; SVE-NEXT: ret539;540; SME2-LABEL: vector_deinterleave_nxv8i16_nxv32i16:541; SME2: // %bb.0:542; SME2-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3543; SME2-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3544; SME2-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3545; SME2-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3546; SME2-NEXT: uzp { z0.h - z3.h }, { z0.h - z3.h }547; SME2-NEXT: ret548 %retval = call {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.vector.deinterleave4.nxv32i16(<vscale x 32 x i16> %vec)549 ret {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} %retval550}551 552define {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} @vector_deinterleave_nxv4i32_nxv16i32(<vscale x 16 x i32> %vec) {553; SVE-LABEL: vector_deinterleave_nxv4i32_nxv16i32:554; SVE: // %bb.0:555; SVE-NEXT: uzp1 z4.s, z2.s, z3.s556; SVE-NEXT: uzp1 z5.s, z0.s, z1.s557; SVE-NEXT: uzp2 z3.s, z2.s, z3.s558; SVE-NEXT: uzp2 z6.s, z0.s, z1.s559; SVE-NEXT: uzp1 z0.s, z5.s, z4.s560; SVE-NEXT: uzp2 z2.s, z5.s, z4.s561; SVE-NEXT: uzp1 z1.s, z6.s, z3.s562; SVE-NEXT: uzp2 z3.s, z6.s, z3.s563; SVE-NEXT: ret564;565; SME2-LABEL: vector_deinterleave_nxv4i32_nxv16i32:566; SME2: // %bb.0:567; SME2-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3568; SME2-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3569; SME2-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3570; SME2-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3571; SME2-NEXT: uzp { z0.s - z3.s }, { z0.s - z3.s }572; SME2-NEXT: ret573 %retval = call {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> %vec)574 ret {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} %retval575}576 577define {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @vector_deinterleave_nxv2i64_nxv8i64(<vscale x 8 x i64> %vec) {578; SVE-LABEL: vector_deinterleave_nxv2i64_nxv8i64:579; SVE: // %bb.0:580; SVE-NEXT: uzp1 z4.d, z2.d, z3.d581; SVE-NEXT: uzp1 z5.d, z0.d, z1.d582; SVE-NEXT: uzp2 z3.d, z2.d, z3.d583; SVE-NEXT: uzp2 z6.d, z0.d, z1.d584; SVE-NEXT: uzp1 z0.d, z5.d, z4.d585; SVE-NEXT: uzp2 z2.d, z5.d, z4.d586; SVE-NEXT: uzp1 z1.d, z6.d, z3.d587; SVE-NEXT: uzp2 z3.d, z6.d, z3.d588; SVE-NEXT: ret589;590; SME2-ALL-LABEL: vector_deinterleave_nxv2i64_nxv8i64:591; SME2-ALL: // %bb.0:592; SME2-ALL-NEXT: uzp { z4.d, z5.d }, z2.d, z3.d593; SME2-ALL-NEXT: uzp { z0.d, z1.d }, z0.d, z1.d594; SME2-ALL-NEXT: uzp { z2.d, z3.d }, z0.d, z4.d595; SME2-ALL-NEXT: uzp { z4.d, z5.d }, z1.d, z5.d596; SME2-ALL-NEXT: mov z0.d, z2.d597; SME2-ALL-NEXT: mov z1.d, z4.d598; SME2-ALL-NEXT: mov z2.d, z3.d599; SME2-ALL-NEXT: mov z3.d, z5.d600; SME2-ALL-NEXT: ret601;602; SME2-256-LABEL: vector_deinterleave_nxv2i64_nxv8i64:603; SME2-256: // %bb.0:604; SME2-256-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3605; SME2-256-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3606; SME2-256-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3607; SME2-256-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3608; SME2-256-NEXT: uzp { z0.d - z3.d }, { z0.d - z3.d }609; SME2-256-NEXT: ret610 %retval = call {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.vector.deinterleave4.nxv8i64(<vscale x 8 x i64> %vec)611 ret {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} %retval612}613 614define {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @vector_deinterleave_nxv2i64_nxv16i64(<vscale x 16 x i64> %vec) {615; SVE-LABEL: vector_deinterleave_nxv2i64_nxv16i64:616; SVE: // %bb.0:617; SVE-NEXT: uzp1 z24.d, z6.d, z7.d618; SVE-NEXT: uzp1 z25.d, z4.d, z5.d619; SVE-NEXT: uzp1 z26.d, z2.d, z3.d620; SVE-NEXT: uzp1 z27.d, z0.d, z1.d621; SVE-NEXT: uzp2 z6.d, z6.d, z7.d622; SVE-NEXT: uzp2 z4.d, z4.d, z5.d623; SVE-NEXT: uzp2 z2.d, z2.d, z3.d624; SVE-NEXT: uzp2 z0.d, z0.d, z1.d625; SVE-NEXT: uzp1 z5.d, z25.d, z24.d626; SVE-NEXT: uzp2 z24.d, z25.d, z24.d627; SVE-NEXT: uzp1 z7.d, z27.d, z26.d628; SVE-NEXT: uzp1 z28.d, z4.d, z6.d629; SVE-NEXT: uzp2 z25.d, z27.d, z26.d630; SVE-NEXT: uzp1 z29.d, z0.d, z2.d631; SVE-NEXT: uzp2 z26.d, z4.d, z6.d632; SVE-NEXT: uzp2 z27.d, z0.d, z2.d633; SVE-NEXT: uzp1 z0.d, z7.d, z5.d634; SVE-NEXT: uzp1 z2.d, z25.d, z24.d635; SVE-NEXT: uzp2 z4.d, z7.d, z5.d636; SVE-NEXT: uzp1 z1.d, z29.d, z28.d637; SVE-NEXT: uzp1 z3.d, z27.d, z26.d638; SVE-NEXT: uzp2 z5.d, z29.d, z28.d639; SVE-NEXT: uzp2 z6.d, z25.d, z24.d640; SVE-NEXT: uzp2 z7.d, z27.d, z26.d641; SVE-NEXT: ret642;643; SME2-ALL-LABEL: vector_deinterleave_nxv2i64_nxv16i64:644; SME2-ALL: // %bb.0:645; SME2-ALL-NEXT: uzp { z6.d, z7.d }, z6.d, z7.d646; SME2-ALL-NEXT: uzp { z24.d, z25.d }, z4.d, z5.d647; SME2-ALL-NEXT: uzp { z26.d, z27.d }, z24.d, z6.d648; SME2-ALL-NEXT: uzp { z2.d, z3.d }, z2.d, z3.d649; SME2-ALL-NEXT: uzp { z0.d, z1.d }, z0.d, z1.d650; SME2-ALL-NEXT: uzp { z28.d, z29.d }, z0.d, z2.d651; SME2-ALL-NEXT: uzp { z4.d, z5.d }, z28.d, z26.d652; SME2-ALL-NEXT: uzp { z30.d, z31.d }, z25.d, z7.d653; SME2-ALL-NEXT: uzp { z0.d, z1.d }, z1.d, z3.d654; SME2-ALL-NEXT: uzp { z6.d, z7.d }, z0.d, z30.d655; SME2-ALL-NEXT: uzp { z24.d, z25.d }, z29.d, z27.d656; SME2-ALL-NEXT: uzp { z26.d, z27.d }, z1.d, z31.d657; SME2-ALL-NEXT: mov z0.d, z4.d658; SME2-ALL-NEXT: mov z1.d, z6.d659; SME2-ALL-NEXT: mov z2.d, z24.d660; SME2-ALL-NEXT: mov z3.d, z26.d661; SME2-ALL-NEXT: mov z4.d, z5.d662; SME2-ALL-NEXT: mov z5.d, z7.d663; SME2-ALL-NEXT: mov z6.d, z25.d664; SME2-ALL-NEXT: mov z7.d, z27.d665; SME2-ALL-NEXT: ret666;667; SME2-256-LABEL: vector_deinterleave_nxv2i64_nxv16i64:668; SME2-256: // %bb.0:669; SME2-256-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3670; SME2-256-NEXT: // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def $z4_z5_z6_z7671; SME2-256-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3672; SME2-256-NEXT: // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def $z4_z5_z6_z7673; SME2-256-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3674; SME2-256-NEXT: // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def $z4_z5_z6_z7675; SME2-256-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3676; SME2-256-NEXT: // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def $z4_z5_z6_z7677; SME2-256-NEXT: uzp { z28.d - z31.d }, { z4.d - z7.d }678; SME2-256-NEXT: uzp { z0.d - z3.d }, { z0.d - z3.d }679; SME2-256-NEXT: uzp { z4.d, z5.d }, z0.d, z28.d680; SME2-256-NEXT: uzp { z6.d, z7.d }, z1.d, z29.d681; SME2-256-NEXT: uzp { z24.d, z25.d }, z2.d, z30.d682; SME2-256-NEXT: uzp { z26.d, z27.d }, z3.d, z31.d683; SME2-256-NEXT: mov z0.d, z4.d684; SME2-256-NEXT: mov z1.d, z6.d685; SME2-256-NEXT: mov z2.d, z24.d686; SME2-256-NEXT: mov z3.d, z26.d687; SME2-256-NEXT: mov z4.d, z5.d688; SME2-256-NEXT: mov z5.d, z7.d689; SME2-256-NEXT: mov z6.d, z25.d690; SME2-256-NEXT: mov z7.d, z27.d691; SME2-256-NEXT: ret692 %retval = call {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.vector.deinterleave8.nxv16i64(<vscale x 16 x i64> %vec)693 ret {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} %retval694}695 696; Predicated697define {<vscale x 16 x i1>, <vscale x 16 x i1>} @vector_deinterleave_nxv16i1_nxv32i1(<vscale x 32 x i1> %vec) {698; CHECK-LABEL: vector_deinterleave_nxv16i1_nxv32i1:699; CHECK: // %bb.0:700; CHECK-NEXT: uzp1 p2.b, p0.b, p1.b701; CHECK-NEXT: uzp2 p1.b, p0.b, p1.b702; CHECK-NEXT: mov p0.b, p2.b703; CHECK-NEXT: ret704 %retval = call {<vscale x 16 x i1>, <vscale x 16 x i1>} @llvm.vector.deinterleave2.nxv32i1(<vscale x 32 x i1> %vec)705 ret {<vscale x 16 x i1>, <vscale x 16 x i1>} %retval706}707 708define {<vscale x 8 x i1>, <vscale x 8 x i1>} @vector_deinterleave_nxv8i1_nxv16i1(<vscale x 16 x i1> %vec) {709; CHECK-LABEL: vector_deinterleave_nxv8i1_nxv16i1:710; CHECK: // %bb.0:711; CHECK-NEXT: uzp1 p1.b, p0.b, p0.b712; CHECK-NEXT: uzp2 p2.b, p0.b, p0.b713; CHECK-NEXT: punpklo p0.h, p1.b714; CHECK-NEXT: punpklo p1.h, p2.b715; CHECK-NEXT: ret716 %retval = call {<vscale x 8 x i1>, <vscale x 8 x i1>} @llvm.vector.deinterleave2.nxv16i1(<vscale x 16 x i1> %vec)717 ret {<vscale x 8 x i1>, <vscale x 8 x i1>} %retval718}719 720define {<vscale x 4 x i1>, <vscale x 4 x i1>} @vector_deinterleave_nxv4i1_nxv8i1(<vscale x 8 x i1> %vec) {721; CHECK-LABEL: vector_deinterleave_nxv4i1_nxv8i1:722; CHECK: // %bb.0:723; CHECK-NEXT: uzp1 p1.h, p0.h, p0.h724; CHECK-NEXT: uzp2 p2.h, p0.h, p0.h725; CHECK-NEXT: punpklo p0.h, p1.b726; CHECK-NEXT: punpklo p1.h, p2.b727; CHECK-NEXT: ret728 %retval = call {<vscale x 4 x i1>, <vscale x 4 x i1>} @llvm.vector.deinterleave2.nxv8i1(<vscale x 8 x i1> %vec)729 ret {<vscale x 4 x i1>, <vscale x 4 x i1>} %retval730}731 732define {<vscale x 2 x i1>, <vscale x 2 x i1>} @vector_deinterleave_nxv2i1_nxv4i1(<vscale x 4 x i1> %vec) {733; CHECK-LABEL: vector_deinterleave_nxv2i1_nxv4i1:734; CHECK: // %bb.0:735; CHECK-NEXT: uzp1 p1.s, p0.s, p0.s736; CHECK-NEXT: uzp2 p2.s, p0.s, p0.s737; CHECK-NEXT: punpklo p0.h, p1.b738; CHECK-NEXT: punpklo p1.h, p2.b739; CHECK-NEXT: ret740 %retval = call {<vscale x 2 x i1>, <vscale x 2 x i1>} @llvm.vector.deinterleave2.nxv4i1(<vscale x 4 x i1> %vec)741 ret {<vscale x 2 x i1>, <vscale x 2 x i1>} %retval742}743 744 745; Split illegal types746 747define {<vscale x 4 x i64>, <vscale x 4 x i64>} @vector_deinterleave_nxv4i64_nxv8i64(<vscale x 8 x i64> %vec) {748; SVE-LABEL: vector_deinterleave_nxv4i64_nxv8i64:749; SVE: // %bb.0:750; SVE-NEXT: uzp1 z4.d, z2.d, z3.d751; SVE-NEXT: uzp1 z5.d, z0.d, z1.d752; SVE-NEXT: uzp2 z6.d, z0.d, z1.d753; SVE-NEXT: uzp2 z3.d, z2.d, z3.d754; SVE-NEXT: mov z0.d, z5.d755; SVE-NEXT: mov z1.d, z4.d756; SVE-NEXT: mov z2.d, z6.d757; SVE-NEXT: ret758;759; SME2-LABEL: vector_deinterleave_nxv4i64_nxv8i64:760; SME2: // %bb.0:761; SME2-NEXT: uzp { z4.d, z5.d }, z0.d, z1.d762; SME2-NEXT: uzp { z6.d, z7.d }, z2.d, z3.d763; SME2-NEXT: mov z0.d, z4.d764; SME2-NEXT: mov z1.d, z6.d765; SME2-NEXT: mov z2.d, z5.d766; SME2-NEXT: mov z3.d, z7.d767; SME2-NEXT: ret768 %retval = call {<vscale x 4 x i64>, <vscale x 4 x i64>} @llvm.vector.deinterleave2.nxv8i64(<vscale x 8 x i64> %vec)769 ret {<vscale x 4 x i64>, <vscale x 4 x i64>} %retval770}771 772define {<vscale x 8 x i64>, <vscale x 8 x i64>} @vector_deinterleave_nxv8i64_nxv16i64(<vscale x 16 x i64> %vec) {773; SVE-LABEL: vector_deinterleave_nxv8i64_nxv16i64:774; SVE: // %bb.0:775; SVE-NEXT: uzp1 z24.d, z2.d, z3.d776; SVE-NEXT: uzp1 z25.d, z0.d, z1.d777; SVE-NEXT: uzp1 z26.d, z4.d, z5.d778; SVE-NEXT: uzp1 z27.d, z6.d, z7.d779; SVE-NEXT: uzp2 z28.d, z0.d, z1.d780; SVE-NEXT: uzp2 z29.d, z2.d, z3.d781; SVE-NEXT: uzp2 z30.d, z4.d, z5.d782; SVE-NEXT: uzp2 z7.d, z6.d, z7.d783; SVE-NEXT: mov z0.d, z25.d784; SVE-NEXT: mov z1.d, z24.d785; SVE-NEXT: mov z2.d, z26.d786; SVE-NEXT: mov z3.d, z27.d787; SVE-NEXT: mov z4.d, z28.d788; SVE-NEXT: mov z5.d, z29.d789; SVE-NEXT: mov z6.d, z30.d790; SVE-NEXT: ret791;792; SME2-LABEL: vector_deinterleave_nxv8i64_nxv16i64:793; SME2: // %bb.0:794; SME2-NEXT: uzp { z24.d, z25.d }, z0.d, z1.d795; SME2-NEXT: uzp { z26.d, z27.d }, z2.d, z3.d796; SME2-NEXT: uzp { z28.d, z29.d }, z4.d, z5.d797; SME2-NEXT: uzp { z30.d, z31.d }, z6.d, z7.d798; SME2-NEXT: mov z0.d, z24.d799; SME2-NEXT: mov z1.d, z26.d800; SME2-NEXT: mov z2.d, z28.d801; SME2-NEXT: mov z3.d, z30.d802; SME2-NEXT: mov z4.d, z25.d803; SME2-NEXT: mov z5.d, z27.d804; SME2-NEXT: mov z6.d, z29.d805; SME2-NEXT: mov z7.d, z31.d806; SME2-NEXT: ret807 %retval = call {<vscale x 8 x i64>, <vscale x 8 x i64>} @llvm.vector.deinterleave2.nxv16i64(<vscale x 16 x i64> %vec)808 ret {<vscale x 8 x i64>, <vscale x 8 x i64>} %retval809}810 811 812; Promote illegal type size813 814define {<vscale x 8 x i8>, <vscale x 8 x i8>} @vector_deinterleave_nxv8i8_nxv16i8(<vscale x 16 x i8> %vec) {815; SVE-LABEL: vector_deinterleave_nxv8i8_nxv16i8:816; SVE: // %bb.0:817; SVE-NEXT: uunpkhi z1.h, z0.b818; SVE-NEXT: uunpklo z2.h, z0.b819; SVE-NEXT: uzp1 z0.h, z2.h, z1.h820; SVE-NEXT: uzp2 z1.h, z2.h, z1.h821; SVE-NEXT: ret822;823; SME2-LABEL: vector_deinterleave_nxv8i8_nxv16i8:824; SME2: // %bb.0:825; SME2-NEXT: uunpkhi z1.h, z0.b826; SME2-NEXT: uunpklo z0.h, z0.b827; SME2-NEXT: uzp { z0.h, z1.h }, z0.h, z1.h828; SME2-NEXT: ret829 %retval = call {<vscale x 8 x i8>, <vscale x 8 x i8>} @llvm.vector.deinterleave2.nxv16i8(<vscale x 16 x i8> %vec)830 ret {<vscale x 8 x i8>, <vscale x 8 x i8>} %retval831}832 833define {<vscale x 4 x i16>, <vscale x 4 x i16>} @vector_deinterleave_nxv4i16_nxv8i16(<vscale x 8 x i16> %vec) {834; SVE-LABEL: vector_deinterleave_nxv4i16_nxv8i16:835; SVE: // %bb.0:836; SVE-NEXT: uunpkhi z1.s, z0.h837; SVE-NEXT: uunpklo z2.s, z0.h838; SVE-NEXT: uzp1 z0.s, z2.s, z1.s839; SVE-NEXT: uzp2 z1.s, z2.s, z1.s840; SVE-NEXT: ret841;842; SME2-LABEL: vector_deinterleave_nxv4i16_nxv8i16:843; SME2: // %bb.0:844; SME2-NEXT: uunpkhi z1.s, z0.h845; SME2-NEXT: uunpklo z0.s, z0.h846; SME2-NEXT: uzp { z0.s, z1.s }, z0.s, z1.s847; SME2-NEXT: ret848 %retval = call {<vscale x 4 x i16>, <vscale x 4 x i16>} @llvm.vector.deinterleave2.nxv8i16(<vscale x 8 x i16> %vec)849 ret {<vscale x 4 x i16>, <vscale x 4 x i16>} %retval850}851 852define {<vscale x 2 x i32>, <vscale x 2 x i32>} @vector_deinterleave_nxv2i32_nxv4i32(<vscale x 4 x i32> %vec) {853; SVE-LABEL: vector_deinterleave_nxv2i32_nxv4i32:854; SVE: // %bb.0:855; SVE-NEXT: uunpkhi z1.d, z0.s856; SVE-NEXT: uunpklo z2.d, z0.s857; SVE-NEXT: uzp1 z0.d, z2.d, z1.d858; SVE-NEXT: uzp2 z1.d, z2.d, z1.d859; SVE-NEXT: ret860;861; SME2-LABEL: vector_deinterleave_nxv2i32_nxv4i32:862; SME2: // %bb.0:863; SME2-NEXT: uunpkhi z1.d, z0.s864; SME2-NEXT: uunpklo z0.d, z0.s865; SME2-NEXT: uzp { z0.d, z1.d }, z0.d, z1.d866; SME2-NEXT: ret867 %retval = call {<vscale x 2 x i32>,<vscale x 2 x i32>} @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %vec)868 ret {<vscale x 2 x i32>, <vscale x 2 x i32>} %retval869}870