933 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mattr=+sve | FileCheck %s -check-prefixes=CHECK,SVE3; RUN: llc < %s -mattr=+sve,+sme2 | FileCheck %s -check-prefixes=CHECK,SVE4; RUN: llc < %s -mattr=+sme2 -force-streaming | FileCheck %s -check-prefixes=CHECK,SME2,SME-ALL5; RUN: llc < %s -mattr=+sme2 -force-streaming -aarch64-sve-vector-bits-min=256 | FileCheck %s -check-prefixes=CHECK,SME2,SME2-2566 7target triple = "aarch64-unknown-linux-gnu"8 9define <vscale x 4 x half> @interleave2_nxv4f16(<vscale x 2 x half> %vec0, <vscale x 2 x half> %vec1) {10; SVE-LABEL: interleave2_nxv4f16:11; SVE: // %bb.0:12; SVE-NEXT: zip2 z2.d, z0.d, z1.d13; SVE-NEXT: zip1 z0.d, z0.d, z1.d14; SVE-NEXT: uzp1 z0.s, z0.s, z2.s15; SVE-NEXT: ret16;17; SME2-LABEL: interleave2_nxv4f16:18; SME2: // %bb.0:19; SME2-NEXT: zip { z0.d, z1.d }, z0.d, z1.d20; SME2-NEXT: uzp1 z0.s, z0.s, z1.s21; SME2-NEXT: ret22 %retval = call <vscale x 4 x half> @llvm.vector.interleave2.nxv4f16(<vscale x 2 x half> %vec0, <vscale x 2 x half> %vec1)23 ret <vscale x 4 x half> %retval24}25 26define <vscale x 8 x half> @interleave2_nxv8f16(<vscale x 4 x half> %vec0, <vscale x 4 x half> %vec1) {27; SVE-LABEL: interleave2_nxv8f16:28; SVE: // %bb.0:29; SVE-NEXT: zip2 z2.s, z0.s, z1.s30; SVE-NEXT: zip1 z0.s, z0.s, z1.s31; SVE-NEXT: uzp1 z0.h, z0.h, z2.h32; SVE-NEXT: ret33;34; SME2-LABEL: interleave2_nxv8f16:35; SME2: // %bb.0:36; SME2-NEXT: zip { z0.s, z1.s }, z0.s, z1.s37; SME2-NEXT: uzp1 z0.h, z0.h, z1.h38; SME2-NEXT: ret39 %retval = call <vscale x 8 x half> @llvm.vector.interleave2.nxv8f16(<vscale x 4 x half> %vec0, <vscale x 4 x half> %vec1)40 ret <vscale x 8 x half> %retval41}42 43define <vscale x 16 x half> @interleave2_nxv16f16(<vscale x 8 x half> %vec0, <vscale x 8 x half> %vec1) {44; SVE-LABEL: interleave2_nxv16f16:45; SVE: // %bb.0:46; SVE-NEXT: zip1 z2.h, z0.h, z1.h47; SVE-NEXT: zip2 z1.h, z0.h, z1.h48; SVE-NEXT: mov z0.d, z2.d49; SVE-NEXT: ret50;51; SME2-LABEL: interleave2_nxv16f16:52; SME2: // %bb.0:53; SME2-NEXT: zip { z0.h, z1.h }, z0.h, z1.h54; SME2-NEXT: ret55 %retval = call <vscale x 16 x half> @llvm.vector.interleave2.nxv16f16(<vscale x 8 x half> %vec0, <vscale x 8 x half> %vec1)56 ret <vscale x 16 x half> %retval57}58 59define <vscale x 4 x float> @interleave2_nxv4f32(<vscale x 2 x float> %vec0, <vscale x 2 x float> %vec1) {60; SVE-LABEL: interleave2_nxv4f32:61; SVE: // %bb.0:62; SVE-NEXT: zip2 z2.d, z0.d, z1.d63; SVE-NEXT: zip1 z0.d, z0.d, z1.d64; SVE-NEXT: uzp1 z0.s, z0.s, z2.s65; SVE-NEXT: ret66;67; SME2-LABEL: interleave2_nxv4f32:68; SME2: // %bb.0:69; SME2-NEXT: zip { z0.d, z1.d }, z0.d, z1.d70; SME2-NEXT: uzp1 z0.s, z0.s, z1.s71; SME2-NEXT: ret72 %retval = call <vscale x 4 x float> @llvm.vector.interleave2.nxv4f32(<vscale x 2 x float> %vec0, <vscale x 2 x float> %vec1)73 ret <vscale x 4 x float> %retval74}75 76define <vscale x 8 x float> @interleave2_nxv8f32(<vscale x 4 x float> %vec0, <vscale x 4 x float> %vec1) {77; SVE-LABEL: interleave2_nxv8f32:78; SVE: // %bb.0:79; SVE-NEXT: zip1 z2.s, z0.s, z1.s80; SVE-NEXT: zip2 z1.s, z0.s, z1.s81; SVE-NEXT: mov z0.d, z2.d82; SVE-NEXT: ret83;84; SME2-LABEL: interleave2_nxv8f32:85; SME2: // %bb.0:86; SME2-NEXT: zip { z0.s, z1.s }, z0.s, z1.s87; SME2-NEXT: ret88 %retval = call <vscale x 8 x float> @llvm.vector.interleave2.nxv8f32(<vscale x 4 x float> %vec0, <vscale x 4 x float> %vec1)89 ret <vscale x 8 x float> %retval90}91 92define <vscale x 4 x double> @interleave2_nxv4f64(<vscale x 2 x double> %vec0, <vscale x 2 x double> %vec1) {93; SVE-LABEL: interleave2_nxv4f64:94; SVE: // %bb.0:95; SVE-NEXT: zip1 z2.d, z0.d, z1.d96; SVE-NEXT: zip2 z1.d, z0.d, z1.d97; SVE-NEXT: mov z0.d, z2.d98; SVE-NEXT: ret99;100; SME2-LABEL: interleave2_nxv4f64:101; SME2: // %bb.0:102; SME2-NEXT: zip { z0.d, z1.d }, z0.d, z1.d103; SME2-NEXT: ret104 %retval = call <vscale x 4 x double>@llvm.vector.interleave2.nxv4f64(<vscale x 2 x double> %vec0, <vscale x 2 x double> %vec1)105 ret <vscale x 4 x double> %retval106}107 108define <vscale x 4 x bfloat> @interleave2_nxv4bf16(<vscale x 2 x bfloat> %vec0, <vscale x 2 x bfloat> %vec1) {109; SVE-LABEL: interleave2_nxv4bf16:110; SVE: // %bb.0:111; SVE-NEXT: zip2 z2.d, z0.d, z1.d112; SVE-NEXT: zip1 z0.d, z0.d, z1.d113; SVE-NEXT: uzp1 z0.s, z0.s, z2.s114; SVE-NEXT: ret115;116; SME2-LABEL: interleave2_nxv4bf16:117; SME2: // %bb.0:118; SME2-NEXT: zip { z0.d, z1.d }, z0.d, z1.d119; SME2-NEXT: uzp1 z0.s, z0.s, z1.s120; SME2-NEXT: ret121 %retval = call <vscale x 4 x bfloat> @llvm.vector.interleave2.nxv4bf16(<vscale x 2 x bfloat> %vec0, <vscale x 2 x bfloat> %vec1)122 ret <vscale x 4 x bfloat> %retval123}124 125define <vscale x 8 x bfloat> @interleave2_nxv8bf16(<vscale x 4 x bfloat> %vec0, <vscale x 4 x bfloat> %vec1) {126; SVE-LABEL: interleave2_nxv8bf16:127; SVE: // %bb.0:128; SVE-NEXT: zip2 z2.s, z0.s, z1.s129; SVE-NEXT: zip1 z0.s, z0.s, z1.s130; SVE-NEXT: uzp1 z0.h, z0.h, z2.h131; SVE-NEXT: ret132;133; SME2-LABEL: interleave2_nxv8bf16:134; SME2: // %bb.0:135; SME2-NEXT: zip { z0.s, z1.s }, z0.s, z1.s136; SME2-NEXT: uzp1 z0.h, z0.h, z1.h137; SME2-NEXT: ret138 %retval = call <vscale x 8 x bfloat> @llvm.vector.interleave2.nxv8bf16(<vscale x 4 x bfloat> %vec0, <vscale x 4 x bfloat> %vec1)139 ret <vscale x 8 x bfloat> %retval140}141 142define <vscale x 16 x bfloat> @interleave2_nxv16bf16(<vscale x 8 x bfloat> %vec0, <vscale x 8 x bfloat> %vec1) {143; SVE-LABEL: interleave2_nxv16bf16:144; SVE: // %bb.0:145; SVE-NEXT: zip1 z2.h, z0.h, z1.h146; SVE-NEXT: zip2 z1.h, z0.h, z1.h147; SVE-NEXT: mov z0.d, z2.d148; SVE-NEXT: ret149;150; SME2-LABEL: interleave2_nxv16bf16:151; SME2: // %bb.0:152; SME2-NEXT: zip { z0.h, z1.h }, z0.h, z1.h153; SME2-NEXT: ret154 %retval = call <vscale x 16 x bfloat> @llvm.vector.interleave2.nxv16bf16(<vscale x 8 x bfloat> %vec0, <vscale x 8 x bfloat> %vec1)155 ret <vscale x 16 x bfloat> %retval156}157 158; Integers159 160define <vscale x 32 x i8> @interleave2_nxv32i8(<vscale x 16 x i8> %vec0, <vscale x 16 x i8> %vec1) {161; SVE-LABEL: interleave2_nxv32i8:162; SVE: // %bb.0:163; SVE-NEXT: zip1 z2.b, z0.b, z1.b164; SVE-NEXT: zip2 z1.b, z0.b, z1.b165; SVE-NEXT: mov z0.d, z2.d166; SVE-NEXT: ret167;168; SME2-LABEL: interleave2_nxv32i8:169; SME2: // %bb.0:170; SME2-NEXT: zip { z0.b, z1.b }, z0.b, z1.b171; SME2-NEXT: ret172 %retval = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> %vec0, <vscale x 16 x i8> %vec1)173 ret <vscale x 32 x i8> %retval174}175 176define <vscale x 16 x i16> @interleave2_nxv16i16(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1) {177; SVE-LABEL: interleave2_nxv16i16:178; SVE: // %bb.0:179; SVE-NEXT: zip1 z2.h, z0.h, z1.h180; SVE-NEXT: zip2 z1.h, z0.h, z1.h181; SVE-NEXT: mov z0.d, z2.d182; SVE-NEXT: ret183;184; SME2-LABEL: interleave2_nxv16i16:185; SME2: // %bb.0:186; SME2-NEXT: zip { z0.h, z1.h }, z0.h, z1.h187; SME2-NEXT: ret188 %retval = call <vscale x 16 x i16> @llvm.vector.interleave2.nxv16i16(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1)189 ret <vscale x 16 x i16> %retval190}191 192define <vscale x 8 x i32> @interleave2_nxv8i32(<vscale x 4 x i32> %vec0, <vscale x 4 x i32> %vec1) {193; SVE-LABEL: interleave2_nxv8i32:194; SVE: // %bb.0:195; SVE-NEXT: zip1 z2.s, z0.s, z1.s196; SVE-NEXT: zip2 z1.s, z0.s, z1.s197; SVE-NEXT: mov z0.d, z2.d198; SVE-NEXT: ret199;200; SME2-LABEL: interleave2_nxv8i32:201; SME2: // %bb.0:202; SME2-NEXT: zip { z0.s, z1.s }, z0.s, z1.s203; SME2-NEXT: ret204 %retval = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %vec0, <vscale x 4 x i32> %vec1)205 ret <vscale x 8 x i32> %retval206}207 208define <vscale x 4 x i64> @interleave2_nxv4i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1) {209; SVE-LABEL: interleave2_nxv4i64:210; SVE: // %bb.0:211; SVE-NEXT: zip1 z2.d, z0.d, z1.d212; SVE-NEXT: zip2 z1.d, z0.d, z1.d213; SVE-NEXT: mov z0.d, z2.d214; SVE-NEXT: ret215;216; SME2-LABEL: interleave2_nxv4i64:217; SME2: // %bb.0:218; SME2-NEXT: zip { z0.d, z1.d }, z0.d, z1.d219; SME2-NEXT: ret220 %retval = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1)221 ret <vscale x 4 x i64> %retval222}223 224define <vscale x 6 x half> @interleave3_nxv6f16(<vscale x 2 x half> %vec0, <vscale x 2 x half> %vec1, <vscale x 2 x half> %vec2) {225; CHECK-LABEL: interleave3_nxv6f16:226; CHECK: // %bb.0:227; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill228; CHECK-NEXT: addvl sp, sp, #-3229; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG230; CHECK-NEXT: .cfi_offset w29, -16231; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2232; CHECK-NEXT: ptrue p0.d233; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2234; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2235; CHECK-NEXT: st3d { z0.d - z2.d }, p0, [sp]236; CHECK-NEXT: ldr z0, [sp, #2, mul vl]237; CHECK-NEXT: ldr z1, [sp, #1, mul vl]238; CHECK-NEXT: ldr z2, [sp]239; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s240; CHECK-NEXT: uzp1 z1.s, z2.s, z1.s241; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h242; CHECK-NEXT: addvl sp, sp, #3243; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload244; CHECK-NEXT: ret245 %retval = call <vscale x 6 x half> @llvm.vector.interleave3.nxv6f16(<vscale x 2 x half> %vec0, <vscale x 2 x half> %vec1, <vscale x 2 x half> %vec2)246 ret <vscale x 6 x half> %retval247}248 249define <vscale x 12 x half> @interleave3_nxv12f16(<vscale x 4 x half> %vec0, <vscale x 4 x half> %vec1, <vscale x 4 x half> %vec2) {250; CHECK-LABEL: interleave3_nxv12f16:251; CHECK: // %bb.0:252; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill253; CHECK-NEXT: addvl sp, sp, #-5254; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x28, 0x1e, 0x22 // sp + 16 + 40 * VG255; CHECK-NEXT: .cfi_offset w29, -16256; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2257; CHECK-NEXT: ptrue p0.s258; CHECK-NEXT: addpl x8, sp, #4259; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2260; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2261; CHECK-NEXT: st3w { z0.s - z2.s }, p0, [sp]262; CHECK-NEXT: ldr z0, [sp, #1, mul vl]263; CHECK-NEXT: ldr z1, [sp]264; CHECK-NEXT: ldr z2, [sp, #2, mul vl]265; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h266; CHECK-NEXT: st1h { z2.s }, p0, [x8, #7, mul vl]267; CHECK-NEXT: str z0, [sp, #3, mul vl]268; CHECK-NEXT: ldr z1, [sp, #4, mul vl]269; CHECK-NEXT: ldr z0, [sp, #3, mul vl]270; CHECK-NEXT: addvl sp, sp, #5271; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload272; CHECK-NEXT: ret273 %retval = call <vscale x 12 x half> @llvm.vector.interleave3.nxv12f16(<vscale x 4 x half> %vec0, <vscale x 4 x half> %vec1, <vscale x 4 x half> %vec2)274 ret <vscale x 12 x half> %retval275}276 277define <vscale x 24 x half> @interleave3_nxv24f16(<vscale x 8 x half> %vec0, <vscale x 8 x half> %vec1, <vscale x 8 x half> %vec2) {278; CHECK-LABEL: interleave3_nxv24f16:279; CHECK: // %bb.0:280; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill281; CHECK-NEXT: addvl sp, sp, #-3282; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG283; CHECK-NEXT: .cfi_offset w29, -16284; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2285; CHECK-NEXT: ptrue p0.h286; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2287; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2288; CHECK-NEXT: st3h { z0.h - z2.h }, p0, [sp]289; CHECK-NEXT: ldr z0, [sp]290; CHECK-NEXT: ldr z1, [sp, #1, mul vl]291; CHECK-NEXT: ldr z2, [sp, #2, mul vl]292; CHECK-NEXT: addvl sp, sp, #3293; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload294; CHECK-NEXT: ret295 %retval = call <vscale x 24 x half> @llvm.vector.interleave3.nxv24f16(<vscale x 8 x half> %vec0, <vscale x 8 x half> %vec1, <vscale x 8 x half> %vec2)296 ret <vscale x 24 x half> %retval297}298 299define <vscale x 6 x float> @interleave3_nxv6f32(<vscale x 2 x float> %vec0, <vscale x 2 x float> %vec1, <vscale x 2 x float> %vec2) {300; CHECK-LABEL: interleave3_nxv6f32:301; CHECK: // %bb.0:302; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill303; CHECK-NEXT: addvl sp, sp, #-5304; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x28, 0x1e, 0x22 // sp + 16 + 40 * VG305; CHECK-NEXT: .cfi_offset w29, -16306; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2307; CHECK-NEXT: ptrue p0.d308; CHECK-NEXT: addpl x8, sp, #4309; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2310; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2311; CHECK-NEXT: st3d { z0.d - z2.d }, p0, [sp]312; CHECK-NEXT: ldr z0, [sp, #1, mul vl]313; CHECK-NEXT: ldr z1, [sp]314; CHECK-NEXT: ldr z2, [sp, #2, mul vl]315; CHECK-NEXT: uzp1 z0.s, z1.s, z0.s316; CHECK-NEXT: st1w { z2.d }, p0, [x8, #7, mul vl]317; CHECK-NEXT: str z0, [sp, #3, mul vl]318; CHECK-NEXT: ldr z1, [sp, #4, mul vl]319; CHECK-NEXT: ldr z0, [sp, #3, mul vl]320; CHECK-NEXT: addvl sp, sp, #5321; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload322; CHECK-NEXT: ret323 %retval = call <vscale x 6 x float> @llvm.vector.interleave3.nxv6f32(<vscale x 2 x float> %vec0, <vscale x 2 x float> %vec1, <vscale x 2 x float> %vec2)324 ret <vscale x 6 x float> %retval325}326 327define <vscale x 12 x float> @interleave3_nxv12f32(<vscale x 4 x float> %vec0, <vscale x 4 x float> %vec1, <vscale x 4 x float> %vec2) {328; CHECK-LABEL: interleave3_nxv12f32:329; CHECK: // %bb.0:330; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill331; CHECK-NEXT: addvl sp, sp, #-3332; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG333; CHECK-NEXT: .cfi_offset w29, -16334; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2335; CHECK-NEXT: ptrue p0.s336; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2337; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2338; CHECK-NEXT: st3w { z0.s - z2.s }, p0, [sp]339; CHECK-NEXT: ldr z0, [sp]340; CHECK-NEXT: ldr z1, [sp, #1, mul vl]341; CHECK-NEXT: ldr z2, [sp, #2, mul vl]342; CHECK-NEXT: addvl sp, sp, #3343; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload344; CHECK-NEXT: ret345 %retval = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32(<vscale x 4 x float> %vec0, <vscale x 4 x float> %vec1, <vscale x 4 x float> %vec2)346 ret <vscale x 12 x float> %retval347}348 349define <vscale x 6 x double> @interleave3_nxv6f64(<vscale x 2 x double> %vec0, <vscale x 2 x double> %vec1, <vscale x 2 x double> %vec2) {350; CHECK-LABEL: interleave3_nxv6f64:351; CHECK: // %bb.0:352; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill353; CHECK-NEXT: addvl sp, sp, #-3354; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG355; CHECK-NEXT: .cfi_offset w29, -16356; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2357; CHECK-NEXT: ptrue p0.d358; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2359; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2360; CHECK-NEXT: st3d { z0.d - z2.d }, p0, [sp]361; CHECK-NEXT: ldr z0, [sp]362; CHECK-NEXT: ldr z1, [sp, #1, mul vl]363; CHECK-NEXT: ldr z2, [sp, #2, mul vl]364; CHECK-NEXT: addvl sp, sp, #3365; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload366; CHECK-NEXT: ret367 %retval = call <vscale x 6 x double>@llvm.vector.interleave3.nxv6f64(<vscale x 2 x double> %vec0, <vscale x 2 x double> %vec1, <vscale x 2 x double> %vec2)368 ret <vscale x 6 x double> %retval369}370 371define <vscale x 6 x bfloat> @interleave3_nxv6bf16(<vscale x 2 x bfloat> %vec0, <vscale x 2 x bfloat> %vec1, <vscale x 2 x bfloat> %vec2) {372; CHECK-LABEL: interleave3_nxv6bf16:373; CHECK: // %bb.0:374; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill375; CHECK-NEXT: addvl sp, sp, #-3376; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG377; CHECK-NEXT: .cfi_offset w29, -16378; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2379; CHECK-NEXT: ptrue p0.d380; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2381; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2382; CHECK-NEXT: st3d { z0.d - z2.d }, p0, [sp]383; CHECK-NEXT: ldr z0, [sp, #2, mul vl]384; CHECK-NEXT: ldr z1, [sp, #1, mul vl]385; CHECK-NEXT: ldr z2, [sp]386; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s387; CHECK-NEXT: uzp1 z1.s, z2.s, z1.s388; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h389; CHECK-NEXT: addvl sp, sp, #3390; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload391; CHECK-NEXT: ret392 %retval = call <vscale x 6 x bfloat> @llvm.vector.interleave3.nxv6bf16(<vscale x 2 x bfloat> %vec0, <vscale x 2 x bfloat> %vec1, <vscale x 2 x bfloat> %vec2)393 ret <vscale x 6 x bfloat> %retval394}395 396define <vscale x 12 x bfloat> @interleave3_nxv12bf16(<vscale x 4 x bfloat> %vec0, <vscale x 4 x bfloat> %vec1, <vscale x 4 x bfloat> %vec2) {397; CHECK-LABEL: interleave3_nxv12bf16:398; CHECK: // %bb.0:399; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill400; CHECK-NEXT: addvl sp, sp, #-5401; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x28, 0x1e, 0x22 // sp + 16 + 40 * VG402; CHECK-NEXT: .cfi_offset w29, -16403; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2404; CHECK-NEXT: ptrue p0.s405; CHECK-NEXT: addpl x8, sp, #4406; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2407; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2408; CHECK-NEXT: st3w { z0.s - z2.s }, p0, [sp]409; CHECK-NEXT: ldr z0, [sp, #1, mul vl]410; CHECK-NEXT: ldr z1, [sp]411; CHECK-NEXT: ldr z2, [sp, #2, mul vl]412; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h413; CHECK-NEXT: st1h { z2.s }, p0, [x8, #7, mul vl]414; CHECK-NEXT: str z0, [sp, #3, mul vl]415; CHECK-NEXT: ldr z1, [sp, #4, mul vl]416; CHECK-NEXT: ldr z0, [sp, #3, mul vl]417; CHECK-NEXT: addvl sp, sp, #5418; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload419; CHECK-NEXT: ret420 %retval = call <vscale x 12 x bfloat> @llvm.vector.interleave3.nxv12bf16(<vscale x 4 x bfloat> %vec0, <vscale x 4 x bfloat> %vec1, <vscale x 4 x bfloat> %vec2)421 ret <vscale x 12 x bfloat> %retval422}423 424define <vscale x 24 x bfloat> @interleave3_nxv24bf16(<vscale x 8 x bfloat> %vec0, <vscale x 8 x bfloat> %vec1, <vscale x 8 x bfloat> %vec2) {425; CHECK-LABEL: interleave3_nxv24bf16:426; CHECK: // %bb.0:427; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill428; CHECK-NEXT: addvl sp, sp, #-3429; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG430; CHECK-NEXT: .cfi_offset w29, -16431; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2432; CHECK-NEXT: ptrue p0.h433; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2434; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2435; CHECK-NEXT: st3h { z0.h - z2.h }, p0, [sp]436; CHECK-NEXT: ldr z0, [sp]437; CHECK-NEXT: ldr z1, [sp, #1, mul vl]438; CHECK-NEXT: ldr z2, [sp, #2, mul vl]439; CHECK-NEXT: addvl sp, sp, #3440; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload441; CHECK-NEXT: ret442 %retval = call <vscale x 24 x bfloat> @llvm.vector.interleave3.nxv24bf16(<vscale x 8 x bfloat> %vec0, <vscale x 8 x bfloat> %vec1, <vscale x 8 x bfloat> %vec2)443 ret <vscale x 24 x bfloat> %retval444}445 446; Integers447 448define <vscale x 48 x i8> @interleave3_nxv48i8(<vscale x 16 x i8> %vec0, <vscale x 16 x i8> %vec1, <vscale x 16 x i8> %vec2) {449; CHECK-LABEL: interleave3_nxv48i8:450; CHECK: // %bb.0:451; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill452; CHECK-NEXT: addvl sp, sp, #-3453; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG454; CHECK-NEXT: .cfi_offset w29, -16455; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2456; CHECK-NEXT: ptrue p0.b457; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2458; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2459; CHECK-NEXT: st3b { z0.b - z2.b }, p0, [sp]460; CHECK-NEXT: ldr z0, [sp]461; CHECK-NEXT: ldr z1, [sp, #1, mul vl]462; CHECK-NEXT: ldr z2, [sp, #2, mul vl]463; CHECK-NEXT: addvl sp, sp, #3464; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload465; CHECK-NEXT: ret466 %retval = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> %vec0, <vscale x 16 x i8> %vec1, <vscale x 16 x i8> %vec2)467 ret <vscale x 48 x i8> %retval468}469 470define <vscale x 24 x i16> @interleave3_nxv24i16(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1, <vscale x 8 x i16> %vec2) {471; CHECK-LABEL: interleave3_nxv24i16:472; CHECK: // %bb.0:473; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill474; CHECK-NEXT: addvl sp, sp, #-3475; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG476; CHECK-NEXT: .cfi_offset w29, -16477; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2478; CHECK-NEXT: ptrue p0.h479; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2480; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2481; CHECK-NEXT: st3h { z0.h - z2.h }, p0, [sp]482; CHECK-NEXT: ldr z0, [sp]483; CHECK-NEXT: ldr z1, [sp, #1, mul vl]484; CHECK-NEXT: ldr z2, [sp, #2, mul vl]485; CHECK-NEXT: addvl sp, sp, #3486; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload487; CHECK-NEXT: ret488 %retval = call <vscale x 24 x i16> @llvm.vector.interleave3.nxv24i16(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1, <vscale x 8 x i16> %vec2)489 ret <vscale x 24 x i16> %retval490}491 492define <vscale x 12 x i32> @interleave3_nxv12i32(<vscale x 4 x i32> %vec0, <vscale x 4 x i32> %vec1, <vscale x 4 x i32> %vec2) {493; CHECK-LABEL: interleave3_nxv12i32:494; CHECK: // %bb.0:495; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill496; CHECK-NEXT: addvl sp, sp, #-3497; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG498; CHECK-NEXT: .cfi_offset w29, -16499; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2500; CHECK-NEXT: ptrue p0.s501; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2502; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2503; CHECK-NEXT: st3w { z0.s - z2.s }, p0, [sp]504; CHECK-NEXT: ldr z0, [sp]505; CHECK-NEXT: ldr z1, [sp, #1, mul vl]506; CHECK-NEXT: ldr z2, [sp, #2, mul vl]507; CHECK-NEXT: addvl sp, sp, #3508; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload509; CHECK-NEXT: ret510 %retval = call <vscale x 12 x i32> @llvm.vector.interleave3.nxv12i32(<vscale x 4 x i32> %vec0, <vscale x 4 x i32> %vec1, <vscale x 4 x i32> %vec2)511 ret <vscale x 12 x i32> %retval512}513 514define <vscale x 6 x i64> @interleave3_nxv6i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1, <vscale x 2 x i64> %vec2) {515; CHECK-LABEL: interleave3_nxv6i64:516; CHECK: // %bb.0:517; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill518; CHECK-NEXT: addvl sp, sp, #-3519; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG520; CHECK-NEXT: .cfi_offset w29, -16521; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2522; CHECK-NEXT: ptrue p0.d523; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2524; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2525; CHECK-NEXT: st3d { z0.d - z2.d }, p0, [sp]526; CHECK-NEXT: ldr z0, [sp]527; CHECK-NEXT: ldr z1, [sp, #1, mul vl]528; CHECK-NEXT: ldr z2, [sp, #2, mul vl]529; CHECK-NEXT: addvl sp, sp, #3530; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload531; CHECK-NEXT: ret532 %retval = call <vscale x 6 x i64> @llvm.vector.interleave3.nxv6i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1, <vscale x 2 x i64> %vec2)533 ret <vscale x 6 x i64> %retval534}535 536define <vscale x 64 x i8> @interleave4_nxv16i8(<vscale x 16 x i8> %vec0, <vscale x 16 x i8> %vec1, <vscale x 16 x i8> %vec2, <vscale x 16 x i8> %vec3) {537; SVE-LABEL: interleave4_nxv16i8:538; SVE: // %bb.0:539; SVE-NEXT: zip1 z4.b, z1.b, z3.b540; SVE-NEXT: zip1 z5.b, z0.b, z2.b541; SVE-NEXT: zip2 z3.b, z1.b, z3.b542; SVE-NEXT: zip2 z6.b, z0.b, z2.b543; SVE-NEXT: zip1 z0.b, z5.b, z4.b544; SVE-NEXT: zip2 z1.b, z5.b, z4.b545; SVE-NEXT: zip1 z2.b, z6.b, z3.b546; SVE-NEXT: zip2 z3.b, z6.b, z3.b547; SVE-NEXT: ret548;549; SME2-LABEL: interleave4_nxv16i8:550; SME2: // %bb.0:551; SME2-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3552; SME2-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3553; SME2-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3554; SME2-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3555; SME2-NEXT: zip { z0.b - z3.b }, { z0.b - z3.b }556; SME2-NEXT: ret557 %retval = call <vscale x 64 x i8> @llvm.vector.interleave4.nxv16i8(<vscale x 16 x i8> %vec0, <vscale x 16 x i8> %vec1, <vscale x 16 x i8> %vec2, <vscale x 16 x i8> %vec3)558 ret <vscale x 64 x i8> %retval559}560 561define <vscale x 32 x i16> @interleave4_nxv8i16(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1, <vscale x 8 x i16> %vec2, <vscale x 8 x i16> %vec3) {562; SVE-LABEL: interleave4_nxv8i16:563; SVE: // %bb.0:564; SVE-NEXT: zip1 z4.h, z1.h, z3.h565; SVE-NEXT: zip1 z5.h, z0.h, z2.h566; SVE-NEXT: zip2 z3.h, z1.h, z3.h567; SVE-NEXT: zip2 z6.h, z0.h, z2.h568; SVE-NEXT: zip1 z0.h, z5.h, z4.h569; SVE-NEXT: zip2 z1.h, z5.h, z4.h570; SVE-NEXT: zip1 z2.h, z6.h, z3.h571; SVE-NEXT: zip2 z3.h, z6.h, z3.h572; SVE-NEXT: ret573;574; SME2-LABEL: interleave4_nxv8i16:575; SME2: // %bb.0:576; SME2-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3577; SME2-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3578; SME2-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3579; SME2-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3580; SME2-NEXT: zip { z0.h - z3.h }, { z0.h - z3.h }581; SME2-NEXT: ret582 %retval = call <vscale x 32 x i16> @llvm.vector.interleave4.nxv32i16(<vscale x 8 x i16> %vec0, <vscale x 8 x i16> %vec1, <vscale x 8 x i16> %vec2, <vscale x 8 x i16> %vec3)583 ret <vscale x 32 x i16> %retval584}585 586define <vscale x 16 x i32> @interleave4_nxv4i32(<vscale x 4 x i32> %vec0, <vscale x 4 x i32> %vec1, <vscale x 4 x i32> %vec2, <vscale x 4 x i32> %vec3) {587; SVE-LABEL: interleave4_nxv4i32:588; SVE: // %bb.0:589; SVE-NEXT: zip1 z4.s, z1.s, z3.s590; SVE-NEXT: zip1 z5.s, z0.s, z2.s591; SVE-NEXT: zip2 z3.s, z1.s, z3.s592; SVE-NEXT: zip2 z6.s, z0.s, z2.s593; SVE-NEXT: zip1 z0.s, z5.s, z4.s594; SVE-NEXT: zip2 z1.s, z5.s, z4.s595; SVE-NEXT: zip1 z2.s, z6.s, z3.s596; SVE-NEXT: zip2 z3.s, z6.s, z3.s597; SVE-NEXT: ret598;599; SME2-LABEL: interleave4_nxv4i32:600; SME2: // %bb.0:601; SME2-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3602; SME2-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3603; SME2-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3604; SME2-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3605; SME2-NEXT: zip { z0.s - z3.s }, { z0.s - z3.s }606; SME2-NEXT: ret607 %retval = call <vscale x 16 x i32> @llvm.vector.interleave4.nxv4i32(<vscale x 4 x i32> %vec0, <vscale x 4 x i32> %vec1, <vscale x 4 x i32> %vec2, <vscale x 4 x i32> %vec3)608 ret <vscale x 16 x i32> %retval609}610 611define <vscale x 8 x i64> @interleave4_nxv8i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1, <vscale x 2 x i64> %vec2, <vscale x 2 x i64> %vec3) {612; SVE-LABEL: interleave4_nxv8i64:613; SVE: // %bb.0:614; SVE-NEXT: zip1 z4.d, z1.d, z3.d615; SVE-NEXT: zip1 z5.d, z0.d, z2.d616; SVE-NEXT: zip2 z3.d, z1.d, z3.d617; SVE-NEXT: zip2 z6.d, z0.d, z2.d618; SVE-NEXT: zip1 z0.d, z5.d, z4.d619; SVE-NEXT: zip2 z1.d, z5.d, z4.d620; SVE-NEXT: zip1 z2.d, z6.d, z3.d621; SVE-NEXT: zip2 z3.d, z6.d, z3.d622; SVE-NEXT: ret623;624; SME-ALL-LABEL: interleave4_nxv8i64:625; SME-ALL: // %bb.0:626; SME-ALL-NEXT: zip { z4.d, z5.d }, z1.d, z3.d627; SME-ALL-NEXT: zip { z2.d, z3.d }, z0.d, z2.d628; SME-ALL-NEXT: zip { z0.d, z1.d }, z2.d, z4.d629; SME-ALL-NEXT: zip { z2.d, z3.d }, z3.d, z5.d630; SME-ALL-NEXT: ret631;632; SME2-256-LABEL: interleave4_nxv8i64:633; SME2-256: // %bb.0:634; SME2-256-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3635; SME2-256-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3636; SME2-256-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3637; SME2-256-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3638; SME2-256-NEXT: zip { z0.d - z3.d }, { z0.d - z3.d }639; SME2-256-NEXT: ret640 %retval = call <vscale x 8 x i64> @llvm.vector.interleave4.nxv8i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1, <vscale x 2 x i64> %vec2, <vscale x 2 x i64> %vec3)641 ret <vscale x 8 x i64> %retval642}643 644define <vscale x 16 x i64> @interleave8_nxv16i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1, <vscale x 2 x i64> %vec2, <vscale x 2 x i64> %vec3, <vscale x 2 x i64> %vec4, <vscale x 2 x i64> %vec5, <vscale x 2 x i64> %vec6, <vscale x 2 x i64> %vec7) {645; SVE-LABEL: interleave8_nxv16i64:646; SVE: // %bb.0:647; SVE-NEXT: zip1 z24.d, z3.d, z7.d648; SVE-NEXT: zip1 z25.d, z1.d, z5.d649; SVE-NEXT: zip1 z26.d, z2.d, z6.d650; SVE-NEXT: zip1 z27.d, z0.d, z4.d651; SVE-NEXT: zip2 z3.d, z3.d, z7.d652; SVE-NEXT: zip2 z1.d, z1.d, z5.d653; SVE-NEXT: zip2 z2.d, z2.d, z6.d654; SVE-NEXT: zip2 z0.d, z0.d, z4.d655; SVE-NEXT: zip1 z4.d, z25.d, z24.d656; SVE-NEXT: zip2 z6.d, z25.d, z24.d657; SVE-NEXT: zip1 z5.d, z27.d, z26.d658; SVE-NEXT: zip2 z7.d, z27.d, z26.d659; SVE-NEXT: zip1 z24.d, z1.d, z3.d660; SVE-NEXT: zip1 z25.d, z0.d, z2.d661; SVE-NEXT: zip2 z26.d, z1.d, z3.d662; SVE-NEXT: zip2 z27.d, z0.d, z2.d663; SVE-NEXT: zip1 z0.d, z5.d, z4.d664; SVE-NEXT: zip2 z1.d, z5.d, z4.d665; SVE-NEXT: zip1 z2.d, z7.d, z6.d666; SVE-NEXT: zip2 z3.d, z7.d, z6.d667; SVE-NEXT: zip1 z4.d, z25.d, z24.d668; SVE-NEXT: zip2 z5.d, z25.d, z24.d669; SVE-NEXT: zip1 z6.d, z27.d, z26.d670; SVE-NEXT: zip2 z7.d, z27.d, z26.d671; SVE-NEXT: ret672;673; SME-ALL-LABEL: interleave8_nxv16i64:674; SME-ALL: // %bb.0:675; SME-ALL-NEXT: zip { z24.d, z25.d }, z3.d, z7.d676; SME-ALL-NEXT: zip { z26.d, z27.d }, z1.d, z5.d677; SME-ALL-NEXT: zip { z28.d, z29.d }, z26.d, z24.d678; SME-ALL-NEXT: zip { z6.d, z7.d }, z2.d, z6.d679; SME-ALL-NEXT: zip { z4.d, z5.d }, z0.d, z4.d680; SME-ALL-NEXT: zip { z2.d, z3.d }, z4.d, z6.d681; SME-ALL-NEXT: zip { z0.d, z1.d }, z2.d, z28.d682; SME-ALL-NEXT: zip { z2.d, z3.d }, z3.d, z29.d683; SME-ALL-NEXT: zip { z24.d, z25.d }, z27.d, z25.d684; SME-ALL-NEXT: zip { z6.d, z7.d }, z5.d, z7.d685; SME-ALL-NEXT: zip { z4.d, z5.d }, z6.d, z24.d686; SME-ALL-NEXT: zip { z6.d, z7.d }, z7.d, z25.d687; SME-ALL-NEXT: ret688;689; SME2-256-LABEL: interleave8_nxv16i64:690; SME2-256: // %bb.0:691; SME2-256-NEXT: zip { z26.d, z27.d }, z3.d, z7.d692; SME2-256-NEXT: zip { z6.d, z7.d }, z2.d, z6.d693; SME2-256-NEXT: zip { z24.d, z25.d }, z1.d, z5.d694; SME2-256-NEXT: zip { z0.d, z1.d }, z0.d, z4.d695; SME2-256-NEXT: mov z28.d, z0.d696; SME2-256-NEXT: mov z29.d, z24.d697; SME2-256-NEXT: mov z30.d, z6.d698; SME2-256-NEXT: mov z31.d, z26.d699; SME2-256-NEXT: mov z24.d, z1.d700; SME2-256-NEXT: mov z26.d, z7.d701; SME2-256-NEXT: zip { z0.d - z3.d }, { z28.d - z31.d }702; SME2-256-NEXT: zip { z4.d - z7.d }, { z24.d - z27.d }703; SME2-256-NEXT: ret704 %retval = call <vscale x 16 x i64> @llvm.vector.interleave8.nxv16i64(<vscale x 2 x i64> %vec0, <vscale x 2 x i64> %vec1, <vscale x 2 x i64> %vec2, <vscale x 2 x i64> %vec3, <vscale x 2 x i64> %vec4, <vscale x 2 x i64> %vec5, <vscale x 2 x i64> %vec6, <vscale x 2 x i64> %vec7)705 ret <vscale x 16 x i64> %retval706}707 708; Predicated709 710define <vscale x 32 x i1> @interleave2_nxv32i1(<vscale x 16 x i1> %vec0, <vscale x 16 x i1> %vec1) {711; CHECK-LABEL: interleave2_nxv32i1:712; CHECK: // %bb.0:713; CHECK-NEXT: zip1 p2.b, p0.b, p1.b714; CHECK-NEXT: zip2 p1.b, p0.b, p1.b715; CHECK-NEXT: mov p0.b, p2.b716; CHECK-NEXT: ret717 %retval = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %vec0, <vscale x 16 x i1> %vec1)718 ret <vscale x 32 x i1> %retval719}720 721define <vscale x 16 x i1> @interleave2_nxv16i1(<vscale x 8 x i1> %vec0, <vscale x 8 x i1> %vec1) {722; CHECK-LABEL: interleave2_nxv16i1:723; CHECK: // %bb.0:724; CHECK-NEXT: zip2 p2.h, p0.h, p1.h725; CHECK-NEXT: zip1 p0.h, p0.h, p1.h726; CHECK-NEXT: uzp1 p0.b, p0.b, p2.b727; CHECK-NEXT: ret728 %retval = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> %vec0, <vscale x 8 x i1> %vec1)729 ret <vscale x 16 x i1> %retval730}731 732define <vscale x 8 x i1> @interleave2_nxv8i1(<vscale x 4 x i1> %vec0, <vscale x 4 x i1> %vec1) {733; CHECK-LABEL: interleave2_nxv8i1:734; CHECK: // %bb.0:735; CHECK-NEXT: zip2 p2.s, p0.s, p1.s736; CHECK-NEXT: zip1 p0.s, p0.s, p1.s737; CHECK-NEXT: uzp1 p0.h, p0.h, p2.h738; CHECK-NEXT: ret739 %retval = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> %vec0, <vscale x 4 x i1> %vec1)740 ret <vscale x 8 x i1> %retval741}742 743define <vscale x 4 x i1> @interleave2_nxv4i1(<vscale x 2 x i1> %vec0, <vscale x 2 x i1> %vec1) {744; CHECK-LABEL: interleave2_nxv4i1:745; CHECK: // %bb.0:746; CHECK-NEXT: zip2 p2.d, p0.d, p1.d747; CHECK-NEXT: zip1 p0.d, p0.d, p1.d748; CHECK-NEXT: uzp1 p0.s, p0.s, p2.s749; CHECK-NEXT: ret750 %retval = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %vec0, <vscale x 2 x i1> %vec1)751 ret <vscale x 4 x i1> %retval752}753 754; Split illegal type size755 756define <vscale x 16 x i32> @interleave2_nxv16i32(<vscale x 8 x i32> %vec0, <vscale x 8 x i32> %vec1) {757; SVE-LABEL: interleave2_nxv16i32:758; SVE: // %bb.0:759; SVE-NEXT: zip1 z4.s, z1.s, z3.s760; SVE-NEXT: zip1 z5.s, z0.s, z2.s761; SVE-NEXT: zip2 z2.s, z0.s, z2.s762; SVE-NEXT: zip2 z3.s, z1.s, z3.s763; SVE-NEXT: mov z0.d, z5.d764; SVE-NEXT: mov z1.d, z2.d765; SVE-NEXT: mov z2.d, z4.d766; SVE-NEXT: ret767;768; SME2-LABEL: interleave2_nxv16i32:769; SME2: // %bb.0:770; SME2-NEXT: zip { z4.s, z5.s }, z0.s, z2.s771; SME2-NEXT: zip { z2.s, z3.s }, z1.s, z3.s772; SME2-NEXT: mov z0.d, z4.d773; SME2-NEXT: mov z1.d, z5.d774; SME2-NEXT: ret775 %retval = call <vscale x 16 x i32>@llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> %vec0, <vscale x 8 x i32> %vec1)776 ret <vscale x 16 x i32> %retval777}778 779define <vscale x 8 x i64> @interleave2_nxv8i64(<vscale x 4 x i64> %vec0, <vscale x 4 x i64> %vec1) {780; SVE-LABEL: interleave2_nxv8i64:781; SVE: // %bb.0:782; SVE-NEXT: zip1 z4.d, z1.d, z3.d783; SVE-NEXT: zip1 z5.d, z0.d, z2.d784; SVE-NEXT: zip2 z2.d, z0.d, z2.d785; SVE-NEXT: zip2 z3.d, z1.d, z3.d786; SVE-NEXT: mov z0.d, z5.d787; SVE-NEXT: mov z1.d, z2.d788; SVE-NEXT: mov z2.d, z4.d789; SVE-NEXT: ret790;791; SME2-LABEL: interleave2_nxv8i64:792; SME2: // %bb.0:793; SME2-NEXT: zip { z4.d, z5.d }, z0.d, z2.d794; SME2-NEXT: zip { z2.d, z3.d }, z1.d, z3.d795; SME2-NEXT: mov z0.d, z4.d796; SME2-NEXT: mov z1.d, z5.d797; SME2-NEXT: ret798 %retval = call <vscale x 8 x i64> @llvm.vector.interleave2.nxv8i64(<vscale x 4 x i64> %vec0, <vscale x 4 x i64> %vec1)799 ret <vscale x 8 x i64> %retval800}801 802; Promote illegal type size803 804define <vscale x 16 x i8> @interleave2_nxv8i8(<vscale x 8 x i8> %vec0, <vscale x 8 x i8> %vec1) {805; SVE-LABEL: interleave2_nxv8i8:806; SVE: // %bb.0:807; SVE-NEXT: zip2 z2.h, z0.h, z1.h808; SVE-NEXT: zip1 z0.h, z0.h, z1.h809; SVE-NEXT: uzp1 z0.b, z0.b, z2.b810; SVE-NEXT: ret811;812; SME2-LABEL: interleave2_nxv8i8:813; SME2: // %bb.0:814; SME2-NEXT: zip { z0.h, z1.h }, z0.h, z1.h815; SME2-NEXT: uzp1 z0.b, z0.b, z1.b816; SME2-NEXT: ret817 %retval = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv16i8(<vscale x 8 x i8> %vec0, <vscale x 8 x i8> %vec1)818 ret <vscale x 16 x i8> %retval819}820 821define <vscale x 8 x i16> @interleave2_nxv4i16(<vscale x 4 x i16> %vec0, <vscale x 4 x i16> %vec1) {822; SVE-LABEL: interleave2_nxv4i16:823; SVE: // %bb.0:824; SVE-NEXT: zip2 z2.s, z0.s, z1.s825; SVE-NEXT: zip1 z0.s, z0.s, z1.s826; SVE-NEXT: uzp1 z0.h, z0.h, z2.h827; SVE-NEXT: ret828;829; SME2-LABEL: interleave2_nxv4i16:830; SME2: // %bb.0:831; SME2-NEXT: zip { z0.s, z1.s }, z0.s, z1.s832; SME2-NEXT: uzp1 z0.h, z0.h, z1.h833; SME2-NEXT: ret834 %retval = call <vscale x 8 x i16> @llvm.vector.interleave2.nxv8i16(<vscale x 4 x i16> %vec0, <vscale x 4 x i16> %vec1)835 ret <vscale x 8 x i16> %retval836}837 838define <vscale x 4 x i32> @interleave2_nxv2i32(<vscale x 2 x i32> %vec0, <vscale x 2 x i32> %vec1) {839; SVE-LABEL: interleave2_nxv2i32:840; SVE: // %bb.0:841; SVE-NEXT: zip2 z2.d, z0.d, z1.d842; SVE-NEXT: zip1 z0.d, z0.d, z1.d843; SVE-NEXT: uzp1 z0.s, z0.s, z2.s844; SVE-NEXT: ret845;846; SME2-LABEL: interleave2_nxv2i32:847; SME2: // %bb.0:848; SME2-NEXT: zip { z0.d, z1.d }, z0.d, z1.d849; SME2-NEXT: uzp1 z0.s, z0.s, z1.s850; SME2-NEXT: ret851 %retval = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %vec0, <vscale x 2 x i32> %vec1)852 ret <vscale x 4 x i32> %retval853}854 855define <vscale x 4 x i16> @interleave2_same_const_splat_nxv4i16() {856; CHECK-LABEL: interleave2_same_const_splat_nxv4i16:857; CHECK: // %bb.0:858; CHECK-NEXT: mov z0.s, #3 // =0x3859; CHECK-NEXT: ret860 %retval = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> splat(i16 3), <vscale x 2 x i16> splat(i16 3))861 ret <vscale x 4 x i16> %retval862}863 864define <vscale x 4 x i16> @interleave2_diff_const_splat_nxv4i16() {865; SVE-LABEL: interleave2_diff_const_splat_nxv4i16:866; SVE: // %bb.0:867; SVE-NEXT: mov z0.d, #4 // =0x4868; SVE-NEXT: mov z1.d, #3 // =0x3869; SVE-NEXT: zip2 z2.d, z1.d, z0.d870; SVE-NEXT: zip1 z0.d, z1.d, z0.d871; SVE-NEXT: uzp1 z0.s, z0.s, z2.s872; SVE-NEXT: ret873;874; SME2-LABEL: interleave2_diff_const_splat_nxv4i16:875; SME2: // %bb.0:876; SME2-NEXT: mov z0.d, #4 // =0x4877; SME2-NEXT: mov z1.d, #3 // =0x3878; SME2-NEXT: zip { z0.d, z1.d }, z1.d, z0.d879; SME2-NEXT: uzp1 z0.s, z0.s, z1.s880; SME2-NEXT: ret881 %retval = call <vscale x 4 x i16> @llvm.vector.interleave2.v4i16(<vscale x 2 x i16> splat(i16 3), <vscale x 2 x i16> splat(i16 4))882 ret <vscale x 4 x i16> %retval883}884 885define <vscale x 4 x i16> @interleave2_same_nonconst_splat_nxv4i16(i16 %a) {886; CHECK-LABEL: interleave2_same_nonconst_splat_nxv4i16:887; CHECK: // %bb.0:888; CHECK-NEXT: mov z0.s, w0889; CHECK-NEXT: ret890 %ins = insertelement <vscale x 2 x i16> poison, i16 %a, i32 0891 %splat = shufflevector <vscale x 2 x i16> %ins, <vscale x 2 x i16> poison, <vscale x 2 x i32> zeroinitializer892 %retval = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %splat, <vscale x 2 x i16> %splat)893 ret <vscale x 4 x i16> %retval894}895 896define <vscale x 4 x i16> @interleave2_diff_nonconst_splat_nxv4i16(i16 %a, i16 %b) {897; SVE-LABEL: interleave2_diff_nonconst_splat_nxv4i16:898; SVE: // %bb.0:899; SVE-NEXT: // kill: def $w1 killed $w1 def $x1900; SVE-NEXT: // kill: def $w0 killed $w0 def $x0901; SVE-NEXT: mov z0.d, x0902; SVE-NEXT: mov z1.d, x1903; SVE-NEXT: zip2 z2.d, z0.d, z1.d904; SVE-NEXT: zip1 z0.d, z0.d, z1.d905; SVE-NEXT: uzp1 z0.s, z0.s, z2.s906; SVE-NEXT: ret907;908; SME2-LABEL: interleave2_diff_nonconst_splat_nxv4i16:909; SME2: // %bb.0:910; SME2-NEXT: // kill: def $w1 killed $w1 def $x1911; SME2-NEXT: // kill: def $w0 killed $w0 def $x0912; SME2-NEXT: mov z0.d, x0913; SME2-NEXT: mov z1.d, x1914; SME2-NEXT: zip { z0.d, z1.d }, z0.d, z1.d915; SME2-NEXT: uzp1 z0.s, z0.s, z1.s916; SME2-NEXT: ret917 %ins1 = insertelement <vscale x 2 x i16> poison, i16 %a, i32 0918 %splat1 = shufflevector <vscale x 2 x i16> %ins1, <vscale x 2 x i16> poison, <vscale x 2 x i32> zeroinitializer919 %ins2 = insertelement <vscale x 2 x i16> poison, i16 %b, i32 0920 %splat2 = shufflevector <vscale x 2 x i16> %ins2, <vscale x 2 x i16> poison, <vscale x 2 x i32> zeroinitializer921 %retval = call <vscale x 4 x i16> @llvm.vector.interleave2.nxv4i16(<vscale x 2 x i16> %splat1, <vscale x 2 x i16> %splat2)922 ret <vscale x 4 x i16> %retval923}924 925define <vscale x 8 x i16> @interleave4_same_const_splat_nxv8i16() {926; CHECK-LABEL: interleave4_same_const_splat_nxv8i16:927; CHECK: // %bb.0:928; CHECK-NEXT: mov z0.h, #3 // =0x3929; CHECK-NEXT: ret930 %retval = call <vscale x 8 x i16> @llvm.vector.interleave4.nxv8i16(<vscale x 2 x i16> splat(i16 3), <vscale x 2 x i16> splat(i16 3), <vscale x 2 x i16> splat(i16 3), <vscale x 2 x i16> splat(i16 3))931 ret <vscale x 8 x i16> %retval932}933