1323 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+bf16 -force-streaming -verify-machineinstrs < %s | FileCheck %s3 4;5; BF/F/S/UMLAL x1 (SINGLE)6;7 8define void @multi_vector_add_single_vg2x1_bf16(i32 %slice, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm) {9; CHECK-LABEL: multi_vector_add_single_vg2x1_bf16:10; CHECK: // %bb.0:11; CHECK-NEXT: mov w8, w012; CHECK-NEXT: bfmlal za.s[w8, 0:1], z0.h, z1.h13; CHECK-NEXT: bfmlal za.s[w8, 14:15], z0.h, z1.h14; CHECK-NEXT: ret15 call void @llvm.aarch64.sme.fmlal.single.vg2x1.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm)16 %slice.14 = add i32 %slice, 1417 call void @llvm.aarch64.sme.fmlal.single.vg2x1.nxv8bf16(i32 %slice.14, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm)18 ret void19}20 21define void @multi_vector_add_single_vg2x1_f16(i32 %slice, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm) {22; CHECK-LABEL: multi_vector_add_single_vg2x1_f16:23; CHECK: // %bb.0:24; CHECK-NEXT: mov w8, w025; CHECK-NEXT: fmlal za.s[w8, 0:1], z0.h, z1.h26; CHECK-NEXT: fmlal za.s[w8, 14:15], z0.h, z1.h27; CHECK-NEXT: ret28 call void @llvm.aarch64.sme.fmlal.single.vg2x1.nxv8f16(i32 %slice, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm)29 %slice.14 = add i32 %slice, 1430 call void @llvm.aarch64.sme.fmlal.single.vg2x1.nxv8f16(i32 %slice.14, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm)31 ret void32}33 34define void @multi_vector_add_single_vg2x1_s16(i32 %slice, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm) {35; CHECK-LABEL: multi_vector_add_single_vg2x1_s16:36; CHECK: // %bb.0:37; CHECK-NEXT: mov w8, w038; CHECK-NEXT: smlal za.s[w8, 0:1], z0.h, z1.h39; CHECK-NEXT: smlal za.s[w8, 14:15], z0.h, z1.h40; CHECK-NEXT: ret41 call void @llvm.aarch64.sme.smlal.single.vg2x1.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm)42 %slice.14 = add i32 %slice, 1443 call void @llvm.aarch64.sme.smlal.single.vg2x1.nxv8i16(i32 %slice.14, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm)44 ret void45}46 47define void @multi_vector_add_single_vg2x1_u16(i32 %slice, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm) {48; CHECK-LABEL: multi_vector_add_single_vg2x1_u16:49; CHECK: // %bb.0:50; CHECK-NEXT: mov w8, w051; CHECK-NEXT: umlal za.s[w8, 0:1], z0.h, z1.h52; CHECK-NEXT: umlal za.s[w8, 14:15], z0.h, z1.h53; CHECK-NEXT: ret54 call void @llvm.aarch64.sme.umlal.single.vg2x1.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm)55 %slice.14 = add i32 %slice, 1456 call void @llvm.aarch64.sme.umlal.single.vg2x1.nxv8i16(i32 %slice.14, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm)57 ret void58}59 60;61; BF/F/S/UMLSL x1 (SINGLE)62;63 64define void @multi_vector_sub_single_vg2x1_bf16(i32 %slice, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm) {65; CHECK-LABEL: multi_vector_sub_single_vg2x1_bf16:66; CHECK: // %bb.0:67; CHECK-NEXT: mov w8, w068; CHECK-NEXT: bfmlsl za.s[w8, 0:1], z0.h, z1.h69; CHECK-NEXT: bfmlsl za.s[w8, 14:15], z0.h, z1.h70; CHECK-NEXT: ret71 call void @llvm.aarch64.sme.fmlsl.single.vg2x1.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm)72 %slice.14 = add i32 %slice, 1473 call void @llvm.aarch64.sme.fmlsl.single.vg2x1.nxv8bf16(i32 %slice.14, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm)74 ret void75}76 77define void @multi_vector_sub_single_vg2x1_f16(i32 %slice, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm) {78; CHECK-LABEL: multi_vector_sub_single_vg2x1_f16:79; CHECK: // %bb.0:80; CHECK-NEXT: mov w8, w081; CHECK-NEXT: fmlsl za.s[w8, 0:1], z0.h, z1.h82; CHECK-NEXT: fmlsl za.s[w8, 14:15], z0.h, z1.h83; CHECK-NEXT: ret84 call void @llvm.aarch64.sme.fmlsl.single.vg2x1.nxv8f16(i32 %slice, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm)85 %slice.14 = add i32 %slice, 1486 call void @llvm.aarch64.sme.fmlsl.single.vg2x1.nxv8f16(i32 %slice.14, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm)87 ret void88}89 90define void @multi_vector_sub_single_vg2x1_s16(i32 %slice, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm) {91; CHECK-LABEL: multi_vector_sub_single_vg2x1_s16:92; CHECK: // %bb.0:93; CHECK-NEXT: mov w8, w094; CHECK-NEXT: smlsl za.s[w8, 0:1], z0.h, z1.h95; CHECK-NEXT: smlsl za.s[w8, 14:15], z0.h, z1.h96; CHECK-NEXT: ret97 call void @llvm.aarch64.sme.smlsl.single.vg2x1.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm)98 %slice.14 = add i32 %slice, 1499 call void @llvm.aarch64.sme.smlsl.single.vg2x1.nxv8i16(i32 %slice.14, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm)100 ret void101}102 103define void @multi_vector_sub_single_vg2x1_u16(i32 %slice, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm) {104; CHECK-LABEL: multi_vector_sub_single_vg2x1_u16:105; CHECK: // %bb.0:106; CHECK-NEXT: mov w8, w0107; CHECK-NEXT: umlsl za.s[w8, 0:1], z0.h, z1.h108; CHECK-NEXT: umlsl za.s[w8, 14:15], z0.h, z1.h109; CHECK-NEXT: ret110 call void @llvm.aarch64.sme.umlsl.single.vg2x1.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm)111 %slice.14 = add i32 %slice, 14112 call void @llvm.aarch64.sme.umlsl.single.vg2x1.nxv8i16(i32 %slice.14, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm)113 ret void114}115 116;117; BF/F/S/UMLAL x2 (SINGLE)118;119 120define void @multi_vector_add_single_vg2x2_bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zm) {121; CHECK-LABEL: multi_vector_add_single_vg2x2_bf16:122; CHECK: // %bb.0:123; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1124; CHECK-NEXT: mov w8, w0125; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1126; CHECK-NEXT: bfmlal za.s[w8, 0:1, vgx2], { z0.h, z1.h }, z2.h127; CHECK-NEXT: bfmlal za.s[w8, 6:7, vgx2], { z0.h, z1.h }, z2.h128; CHECK-NEXT: ret129 call void @llvm.aarch64.sme.fmlal.single.vg2x2.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zm)130 %slice.6 = add i32 %slice, 6131 call void @llvm.aarch64.sme.fmlal.single.vg2x2.nxv8bf16(i32 %slice.6, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zm)132 ret void133}134 135define void @multi_vector_add_single_vg2x2_f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zm) {136; CHECK-LABEL: multi_vector_add_single_vg2x2_f16:137; CHECK: // %bb.0:138; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1139; CHECK-NEXT: mov w8, w0140; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1141; CHECK-NEXT: fmlal za.s[w8, 0:1, vgx2], { z0.h, z1.h }, z2.h142; CHECK-NEXT: fmlal za.s[w8, 6:7, vgx2], { z0.h, z1.h }, z2.h143; CHECK-NEXT: ret144 call void @llvm.aarch64.sme.fmlal.single.vg2x2.nxv8f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zm)145 %slice.6 = add i32 %slice, 6146 call void @llvm.aarch64.sme.fmlal.single.vg2x2.nxv8f16(i32 %slice.6, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zm)147 ret void148}149 150define void @multi_vector_add_single_vg2x2_s16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm) {151; CHECK-LABEL: multi_vector_add_single_vg2x2_s16:152; CHECK: // %bb.0:153; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1154; CHECK-NEXT: mov w8, w0155; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1156; CHECK-NEXT: smlal za.s[w8, 0:1, vgx2], { z0.h, z1.h }, z2.h157; CHECK-NEXT: smlal za.s[w8, 6:7, vgx2], { z0.h, z1.h }, z2.h158; CHECK-NEXT: ret159 call void @llvm.aarch64.sme.smlal.single.vg2x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm)160 %slice.6 = add i32 %slice, 6161 call void @llvm.aarch64.sme.smlal.single.vg2x2.nxv8i16(i32 %slice.6, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm)162 ret void163}164 165define void @multi_vector_add_single_vg2x2_u16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm) {166; CHECK-LABEL: multi_vector_add_single_vg2x2_u16:167; CHECK: // %bb.0:168; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1169; CHECK-NEXT: mov w8, w0170; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1171; CHECK-NEXT: umlal za.s[w8, 0:1, vgx2], { z0.h, z1.h }, z2.h172; CHECK-NEXT: umlal za.s[w8, 6:7, vgx2], { z0.h, z1.h }, z2.h173; CHECK-NEXT: ret174 call void @llvm.aarch64.sme.umlal.single.vg2x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm)175 %slice.6 = add i32 %slice, 6176 call void @llvm.aarch64.sme.umlal.single.vg2x2.nxv8i16(i32 %slice.6, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm)177 ret void178}179 180;181; BF/F/S/UMLSL x2 (SINGLE)182;183 184define void @multi_vector_sub_single_vg2x2_bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zm) {185; CHECK-LABEL: multi_vector_sub_single_vg2x2_bf16:186; CHECK: // %bb.0:187; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1188; CHECK-NEXT: mov w8, w0189; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1190; CHECK-NEXT: bfmlsl za.s[w8, 0:1, vgx2], { z0.h, z1.h }, z2.h191; CHECK-NEXT: bfmlsl za.s[w8, 6:7, vgx2], { z0.h, z1.h }, z2.h192; CHECK-NEXT: ret193 call void @llvm.aarch64.sme.fmlsl.single.vg2x2.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zm)194 %slice.6 = add i32 %slice, 6195 call void @llvm.aarch64.sme.fmlsl.single.vg2x2.nxv8bf16(i32 %slice.6, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zm)196 ret void197}198 199define void @multi_vector_sub_single_vg2x2_f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zm) {200; CHECK-LABEL: multi_vector_sub_single_vg2x2_f16:201; CHECK: // %bb.0:202; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1203; CHECK-NEXT: mov w8, w0204; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1205; CHECK-NEXT: fmlsl za.s[w8, 0:1, vgx2], { z0.h, z1.h }, z2.h206; CHECK-NEXT: fmlsl za.s[w8, 6:7, vgx2], { z0.h, z1.h }, z2.h207; CHECK-NEXT: ret208 call void @llvm.aarch64.sme.fmlsl.single.vg2x2.nxv8f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zm)209 %slice.6 = add i32 %slice, 6210 call void @llvm.aarch64.sme.fmlsl.single.vg2x2.nxv8f16(i32 %slice.6, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zm)211 ret void212}213 214define void @multi_vector_sub_single_vg2x2_s16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm) {215; CHECK-LABEL: multi_vector_sub_single_vg2x2_s16:216; CHECK: // %bb.0:217; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1218; CHECK-NEXT: mov w8, w0219; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1220; CHECK-NEXT: smlsl za.s[w8, 0:1, vgx2], { z0.h, z1.h }, z2.h221; CHECK-NEXT: smlsl za.s[w8, 6:7, vgx2], { z0.h, z1.h }, z2.h222; CHECK-NEXT: ret223 call void @llvm.aarch64.sme.smlsl.single.vg2x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm)224 %slice.6 = add i32 %slice, 6225 call void @llvm.aarch64.sme.smlsl.single.vg2x2.nxv8i16(i32 %slice.6, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm)226 ret void227}228 229define void @multi_vector_sub_single_vg2x2_u16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm) {230; CHECK-LABEL: multi_vector_sub_single_vg2x2_u16:231; CHECK: // %bb.0:232; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1233; CHECK-NEXT: mov w8, w0234; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1235; CHECK-NEXT: umlsl za.s[w8, 0:1, vgx2], { z0.h, z1.h }, z2.h236; CHECK-NEXT: umlsl za.s[w8, 6:7, vgx2], { z0.h, z1.h }, z2.h237; CHECK-NEXT: ret238 call void @llvm.aarch64.sme.umlsl.single.vg2x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm)239 %slice.6 = add i32 %slice, 6240 call void @llvm.aarch64.sme.umlsl.single.vg2x2.nxv8i16(i32 %slice.6, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm)241 ret void242}243 244;245; BF/F/S/UMLAL x4 (SINGLE)246;247 248define void @multi_vector_add_single_vg2x4_bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3, <vscale x 8 x bfloat> %zm) {249; CHECK-LABEL: multi_vector_add_single_vg2x4_bf16:250; CHECK: // %bb.0:251; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3252; CHECK-NEXT: mov w8, w0253; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3254; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3255; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3256; CHECK-NEXT: bfmlal za.s[w8, 0:1, vgx4], { z0.h - z3.h }, z4.h257; CHECK-NEXT: bfmlal za.s[w8, 6:7, vgx4], { z0.h - z3.h }, z4.h258; CHECK-NEXT: ret259 call void @llvm.aarch64.sme.fmlal.single.vg2x4.nxv8bf16(i32 %slice,260 <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3,261 <vscale x 8 x bfloat> %zm)262 %slice.6 = add i32 %slice, 6263 call void @llvm.aarch64.sme.fmlal.single.vg2x4.nxv8bf16(i32 %slice.6,264 <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3,265 <vscale x 8 x bfloat> %zm)266 ret void267}268 269define void @multi_vector_add_single_vg2x4_f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3, <vscale x 8 x half> %zm) {270; CHECK-LABEL: multi_vector_add_single_vg2x4_f16:271; CHECK: // %bb.0:272; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3273; CHECK-NEXT: mov w8, w0274; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3275; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3276; CHECK-NEXT: mov z3.d, z2.d277; CHECK-NEXT: fmlal za.s[w8, 0:1, vgx4], { z0.h - z3.h }, z4.h278; CHECK-NEXT: fmlal za.s[w8, 6:7, vgx4], { z0.h - z3.h }, z4.h279; CHECK-NEXT: ret280 call void @llvm.aarch64.sme.fmlal.single.vg2x4.nxv8f16(i32 %slice,281 <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn2,282 <vscale x 8 x half> %zm)283 %slice.6 = add i32 %slice, 6284 call void @llvm.aarch64.sme.fmlal.single.vg2x4.nxv8f16(i32 %slice.6,285 <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn2,286 <vscale x 8 x half> %zm)287 ret void288}289 290define void @multi_vector_add_single_vg2x4_s16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zm) {291; CHECK-LABEL: multi_vector_add_single_vg2x4_s16:292; CHECK: // %bb.0:293; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3294; CHECK-NEXT: mov w8, w0295; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3296; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3297; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3298; CHECK-NEXT: smlal za.s[w8, 0:1, vgx4], { z0.h - z3.h }, z4.h299; CHECK-NEXT: smlal za.s[w8, 6:7, vgx4], { z0.h - z3.h }, z4.h300; CHECK-NEXT: ret301 call void @llvm.aarch64.sme.smlal.single.vg2x4.nxv8i16(i32 %slice,302 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,303 <vscale x 8 x i16> %zm)304 %slice.6 = add i32 %slice, 6305 call void @llvm.aarch64.sme.smlal.single.vg2x4.nxv8i16(i32 %slice.6,306 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,307 <vscale x 8 x i16> %zm)308 ret void309}310 311define void @multi_vector_add_single_vg2x4_u16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zm) {312; CHECK-LABEL: multi_vector_add_single_vg2x4_u16:313; CHECK: // %bb.0:314; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3315; CHECK-NEXT: mov w8, w0316; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3317; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3318; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3319; CHECK-NEXT: umlal za.s[w8, 0:1, vgx4], { z0.h - z3.h }, z4.h320; CHECK-NEXT: umlal za.s[w8, 6:7, vgx4], { z0.h - z3.h }, z4.h321; CHECK-NEXT: ret322 call void @llvm.aarch64.sme.umlal.single.vg2x4.nxv8i16(i32 %slice,323 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,324 <vscale x 8 x i16> %zm)325 %slice.6 = add i32 %slice, 6326 call void @llvm.aarch64.sme.umlal.single.vg2x4.nxv8i16(i32 %slice.6,327 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,328 <vscale x 8 x i16> %zm)329 ret void330}331 332;333; BF/F/S/UMLSL x4 (SINGLE)334;335 336define void @multi_vector_sub_single_vg2x4_bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3, <vscale x 8 x bfloat> %zm) {337; CHECK-LABEL: multi_vector_sub_single_vg2x4_bf16:338; CHECK: // %bb.0:339; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3340; CHECK-NEXT: mov w8, w0341; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3342; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3343; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3344; CHECK-NEXT: bfmlsl za.s[w8, 0:1, vgx4], { z0.h - z3.h }, z4.h345; CHECK-NEXT: bfmlsl za.s[w8, 6:7, vgx4], { z0.h - z3.h }, z4.h346; CHECK-NEXT: ret347 call void @llvm.aarch64.sme.fmlsl.single.vg2x4.nxv8bf16(i32 %slice,348 <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3,349 <vscale x 8 x bfloat> %zm)350 %slice.6 = add i32 %slice, 6351 call void @llvm.aarch64.sme.fmlsl.single.vg2x4.nxv8bf16(i32 %slice.6,352 <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3,353 <vscale x 8 x bfloat> %zm)354 ret void355}356 357define void @multi_vector_sub_single_vg2x4_f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3, <vscale x 8 x half> %zm) {358; CHECK-LABEL: multi_vector_sub_single_vg2x4_f16:359; CHECK: // %bb.0:360; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3361; CHECK-NEXT: mov w8, w0362; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3363; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3364; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3365; CHECK-NEXT: fmlsl za.s[w8, 0:1, vgx4], { z0.h - z3.h }, z4.h366; CHECK-NEXT: fmlsl za.s[w8, 6:7, vgx4], { z0.h - z3.h }, z4.h367; CHECK-NEXT: ret368 call void @llvm.aarch64.sme.fmlsl.single.vg2x4.nxv8f16(i32 %slice,369 <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3,370 <vscale x 8 x half> %zm)371 %slice.6 = add i32 %slice, 6372 call void @llvm.aarch64.sme.fmlsl.single.vg2x4.nxv8f16(i32 %slice.6,373 <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3,374 <vscale x 8 x half> %zm)375 ret void376}377 378define void @multi_vector_sub_single_vg2x4_s16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zm) {379; CHECK-LABEL: multi_vector_sub_single_vg2x4_s16:380; CHECK: // %bb.0:381; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3382; CHECK-NEXT: mov w8, w0383; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3384; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3385; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3386; CHECK-NEXT: smlsl za.s[w8, 0:1, vgx4], { z0.h - z3.h }, z4.h387; CHECK-NEXT: smlsl za.s[w8, 6:7, vgx4], { z0.h - z3.h }, z4.h388; CHECK-NEXT: ret389 call void @llvm.aarch64.sme.smlsl.single.vg2x4.nxv8i16(i32 %slice,390 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,391 <vscale x 8 x i16> %zm)392 %slice.6 = add i32 %slice, 6393 call void @llvm.aarch64.sme.smlsl.single.vg2x4.nxv8i16(i32 %slice.6,394 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,395 <vscale x 8 x i16> %zm)396 ret void397}398 399define void @multi_vector_sub_single_vg2x4_u16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zm) {400; CHECK-LABEL: multi_vector_sub_single_vg2x4_u16:401; CHECK: // %bb.0:402; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3403; CHECK-NEXT: mov w8, w0404; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3405; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3406; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3407; CHECK-NEXT: umlsl za.s[w8, 0:1, vgx4], { z0.h - z3.h }, z4.h408; CHECK-NEXT: umlsl za.s[w8, 6:7, vgx4], { z0.h - z3.h }, z4.h409; CHECK-NEXT: ret410 call void @llvm.aarch64.sme.umlsl.single.vg2x4.nxv8i16(i32 %slice,411 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,412 <vscale x 8 x i16> %zm)413 %slice.6 = add i32 %slice, 6414 call void @llvm.aarch64.sme.umlsl.single.vg2x4.nxv8i16(i32 %slice.6,415 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,416 <vscale x 8 x i16> %zm)417 ret void418}419 420;421; BF/F/S/UMLAL x2 (MULTI)422;423 424define void @multi_vector_add_multi_vg2x2_bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zm0, <vscale x 8 x bfloat> %zm1) {425; CHECK-LABEL: multi_vector_add_multi_vg2x2_bf16:426; CHECK: // %bb.0:427; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z2_z3 def $z2_z3428; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1429; CHECK-NEXT: mov w8, w0430; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z2_z3 def $z2_z3431; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1432; CHECK-NEXT: bfmlal za.s[w8, 0:1, vgx2], { z0.h, z1.h }, { z2.h, z3.h }433; CHECK-NEXT: bfmlal za.s[w8, 6:7, vgx2], { z0.h, z1.h }, { z2.h, z3.h }434; CHECK-NEXT: ret435 call void @llvm.aarch64.sme.fmlal.vg2x2.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,436 <vscale x 8 x bfloat> %zm0, <vscale x 8 x bfloat> %zm1)437 %slice.6 = add i32 %slice, 6438 call void @llvm.aarch64.sme.fmlal.vg2x2.nxv8bf16(i32 %slice.6, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,439 <vscale x 8 x bfloat> %zm0, <vscale x 8 x bfloat> %zm1)440 ret void441}442 443define void @multi_vector_add_multi_vg2x2_f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zm0, <vscale x 8 x half> %zm1) {444; CHECK-LABEL: multi_vector_add_multi_vg2x2_f16:445; CHECK: // %bb.0:446; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z2_z3 def $z2_z3447; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1448; CHECK-NEXT: mov w8, w0449; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z2_z3 def $z2_z3450; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1451; CHECK-NEXT: fmlal za.s[w8, 0:1, vgx2], { z0.h, z1.h }, { z2.h, z3.h }452; CHECK-NEXT: fmlal za.s[w8, 6:7, vgx2], { z0.h, z1.h }, { z2.h, z3.h }453; CHECK-NEXT: ret454 call void @llvm.aarch64.sme.fmlal.vg2x2.nxv8f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,455 <vscale x 8 x half> %zm0, <vscale x 8 x half> %zm1)456 %slice.6 = add i32 %slice, 6457 call void @llvm.aarch64.sme.fmlal.vg2x2.nxv8f16(i32 %slice.6, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,458 <vscale x 8 x half> %zm0, <vscale x 8 x half> %zm1)459 ret void460}461 462define void @multi_vector_add_multi_vg2x2_s16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1) {463; CHECK-LABEL: multi_vector_add_multi_vg2x2_s16:464; CHECK: // %bb.0:465; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z2_z3 def $z2_z3466; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1467; CHECK-NEXT: mov w8, w0468; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z2_z3 def $z2_z3469; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1470; CHECK-NEXT: smlal za.s[w8, 0:1, vgx2], { z0.h, z1.h }, { z2.h, z3.h }471; CHECK-NEXT: smlal za.s[w8, 6:7, vgx2], { z0.h, z1.h }, { z2.h, z3.h }472; CHECK-NEXT: ret473 call void @llvm.aarch64.sme.smlal.vg2x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1,474 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1)475 %slice.6 = add i32 %slice, 6476 call void @llvm.aarch64.sme.smlal.vg2x2.nxv8i16(i32 %slice.6, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1,477 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1)478 ret void479}480 481define void @multi_vector_add_multi_vg2x2_u16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1) {482; CHECK-LABEL: multi_vector_add_multi_vg2x2_u16:483; CHECK: // %bb.0:484; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z2_z3 def $z2_z3485; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1486; CHECK-NEXT: mov w8, w0487; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z2_z3 def $z2_z3488; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1489; CHECK-NEXT: umlal za.s[w8, 0:1, vgx2], { z0.h, z1.h }, { z2.h, z3.h }490; CHECK-NEXT: umlal za.s[w8, 6:7, vgx2], { z0.h, z1.h }, { z2.h, z3.h }491; CHECK-NEXT: ret492 call void @llvm.aarch64.sme.umlal.vg2x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1,493 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1)494 %slice.6 = add i32 %slice, 6495 call void @llvm.aarch64.sme.umlal.vg2x2.nxv8i16(i32 %slice.6, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1,496 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1)497 ret void498}499 500;501; BF/F/S/UMLSL x2 (MULTI)502;503 504define void @multi_vector_sub_multi_vg2x2_bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zm0, <vscale x 8 x bfloat> %zm1) {505; CHECK-LABEL: multi_vector_sub_multi_vg2x2_bf16:506; CHECK: // %bb.0:507; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z2_z3 def $z2_z3508; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1509; CHECK-NEXT: mov w8, w0510; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z2_z3 def $z2_z3511; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1512; CHECK-NEXT: bfmlsl za.s[w8, 0:1, vgx2], { z0.h, z1.h }, { z2.h, z3.h }513; CHECK-NEXT: bfmlsl za.s[w8, 6:7, vgx2], { z0.h, z1.h }, { z2.h, z3.h }514; CHECK-NEXT: ret515 call void @llvm.aarch64.sme.fmlsl.vg2x2.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,516 <vscale x 8 x bfloat> %zm0, <vscale x 8 x bfloat> %zm1)517 %slice.6 = add i32 %slice, 6518 call void @llvm.aarch64.sme.fmlsl.vg2x2.nxv8bf16(i32 %slice.6, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1,519 <vscale x 8 x bfloat> %zm0, <vscale x 8 x bfloat> %zm1)520 ret void521}522 523define void @multi_vector_sub_multi_vg2x2_f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zm0, <vscale x 8 x half> %zm1) {524; CHECK-LABEL: multi_vector_sub_multi_vg2x2_f16:525; CHECK: // %bb.0:526; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z2_z3 def $z2_z3527; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1528; CHECK-NEXT: mov w8, w0529; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z2_z3 def $z2_z3530; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1531; CHECK-NEXT: fmlsl za.s[w8, 0:1, vgx2], { z0.h, z1.h }, { z2.h, z3.h }532; CHECK-NEXT: fmlsl za.s[w8, 6:7, vgx2], { z0.h, z1.h }, { z2.h, z3.h }533; CHECK-NEXT: ret534 call void @llvm.aarch64.sme.fmlsl.vg2x2.nxv8f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,535 <vscale x 8 x half> %zm0, <vscale x 8 x half> %zm1)536 %slice.6 = add i32 %slice, 6537 call void @llvm.aarch64.sme.fmlsl.vg2x2.nxv8f16(i32 %slice.6, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1,538 <vscale x 8 x half> %zm0, <vscale x 8 x half> %zm1)539 ret void540}541 542define void @multi_vector_sub_multi_vg2x2_s16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1) {543; CHECK-LABEL: multi_vector_sub_multi_vg2x2_s16:544; CHECK: // %bb.0:545; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z2_z3 def $z2_z3546; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1547; CHECK-NEXT: mov w8, w0548; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z2_z3 def $z2_z3549; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1550; CHECK-NEXT: smlsl za.s[w8, 0:1, vgx2], { z0.h, z1.h }, { z2.h, z3.h }551; CHECK-NEXT: smlsl za.s[w8, 6:7, vgx2], { z0.h, z1.h }, { z2.h, z3.h }552; CHECK-NEXT: ret553 call void @llvm.aarch64.sme.smlsl.vg2x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1,554 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1)555 %slice.6 = add i32 %slice, 6556 call void @llvm.aarch64.sme.smlsl.vg2x2.nxv8i16(i32 %slice.6, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1,557 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1)558 ret void559}560 561define void @multi_vector_sub_multi_vg2x2_u16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1) {562; CHECK-LABEL: multi_vector_sub_multi_vg2x2_u16:563; CHECK: // %bb.0:564; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z2_z3 def $z2_z3565; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1566; CHECK-NEXT: mov w8, w0567; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z2_z3 def $z2_z3568; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1569; CHECK-NEXT: umlsl za.s[w8, 0:1, vgx2], { z0.h, z1.h }, { z2.h, z3.h }570; CHECK-NEXT: umlsl za.s[w8, 6:7, vgx2], { z0.h, z1.h }, { z2.h, z3.h }571; CHECK-NEXT: ret572 call void @llvm.aarch64.sme.umlsl.vg2x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1,573 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1)574 %slice.6 = add i32 %slice, 6575 call void @llvm.aarch64.sme.umlsl.vg2x2.nxv8i16(i32 %slice.6, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1,576 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1)577 ret void578}579 580;581; BF/F/S/UMLAL x4 (MULTI)582;583 584define void @multi_vector_add_multi_vg2x4_bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3,585; CHECK-LABEL: multi_vector_add_multi_vg2x4_bf16:586; CHECK: // %bb.0:587; CHECK-NEXT: // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def $z4_z5_z6_z7588; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3589; CHECK-NEXT: mov w8, w0590; CHECK-NEXT: // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def $z4_z5_z6_z7591; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3592; CHECK-NEXT: // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def $z4_z5_z6_z7593; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3594; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def $z4_z5_z6_z7595; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3596; CHECK-NEXT: bfmlal za.s[w8, 0:1, vgx4], { z0.h - z3.h }, { z4.h - z7.h }597; CHECK-NEXT: bfmlal za.s[w8, 6:7, vgx4], { z0.h - z3.h }, { z4.h - z7.h }598; CHECK-NEXT: ret599 <vscale x 8 x bfloat> %zm0, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2, <vscale x 8 x bfloat> %zm3) {600 call void @llvm.aarch64.sme.fmlal.vg2x4.nxv8bf16(i32 %slice,601 <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3,602 <vscale x 8 x bfloat> %zm0, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2, <vscale x 8 x bfloat> %zm3)603 %slice.6 = add i32 %slice, 6604 call void @llvm.aarch64.sme.fmlal.vg2x4.nxv8bf16(i32 %slice.6,605 <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3,606 <vscale x 8 x bfloat> %zm0, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2, <vscale x 8 x bfloat> %zm3)607 ret void608}609 610define void @multi_vector_add_multi_vg2x4_f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3,611; CHECK-LABEL: multi_vector_add_multi_vg2x4_f16:612; CHECK: // %bb.0:613; CHECK-NEXT: // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def $z4_z5_z6_z7614; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3615; CHECK-NEXT: mov w8, w0616; CHECK-NEXT: // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def $z4_z5_z6_z7617; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3618; CHECK-NEXT: // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def $z4_z5_z6_z7619; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3620; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def $z4_z5_z6_z7621; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3622; CHECK-NEXT: fmlal za.s[w8, 0:1, vgx4], { z0.h - z3.h }, { z4.h - z7.h }623; CHECK-NEXT: fmlal za.s[w8, 6:7, vgx4], { z0.h - z3.h }, { z4.h - z7.h }624; CHECK-NEXT: ret625 <vscale x 8 x half> %zm0, <vscale x 8 x half> %zm1, <vscale x 8 x half> %zm2, <vscale x 8 x half> %zm3) {626 call void @llvm.aarch64.sme.fmlal.vg2x4.nxv8f16(i32 %slice,627 <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3,628 <vscale x 8 x half> %zm0, <vscale x 8 x half> %zm1, <vscale x 8 x half> %zm2, <vscale x 8 x half> %zm3)629 %slice.6 = add i32 %slice, 6630 call void @llvm.aarch64.sme.fmlal.vg2x4.nxv8f16(i32 %slice.6,631 <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3,632 <vscale x 8 x half> %zm0, <vscale x 8 x half> %zm1, <vscale x 8 x half> %zm2, <vscale x 8 x half> %zm3)633 ret void634}635 636define void @multi_vector_add_multi_vg2x4_s16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,637; CHECK-LABEL: multi_vector_add_multi_vg2x4_s16:638; CHECK: // %bb.0:639; CHECK-NEXT: // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def $z4_z5_z6_z7640; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3641; CHECK-NEXT: mov w8, w0642; CHECK-NEXT: // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def $z4_z5_z6_z7643; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3644; CHECK-NEXT: // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def $z4_z5_z6_z7645; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3646; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def $z4_z5_z6_z7647; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3648; CHECK-NEXT: smlal za.s[w8, 0:1, vgx4], { z0.h - z3.h }, { z4.h - z7.h }649; CHECK-NEXT: smlal za.s[w8, 6:7, vgx4], { z0.h - z3.h }, { z4.h - z7.h }650; CHECK-NEXT: ret651 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2, <vscale x 8 x i16> %zm3) {652 call void @llvm.aarch64.sme.smlal.vg2x4.nxv8i16(i32 %slice,653 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,654 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2, <vscale x 8 x i16> %zm3)655 %slice.6 = add i32 %slice, 6656 call void @llvm.aarch64.sme.smlal.vg2x4.nxv8i16(i32 %slice.6,657 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,658 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2, <vscale x 8 x i16> %zm3)659 ret void660}661 662define void @multi_vector_add_multi_vg2x4_u16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,663; CHECK-LABEL: multi_vector_add_multi_vg2x4_u16:664; CHECK: // %bb.0:665; CHECK-NEXT: // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def $z4_z5_z6_z7666; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3667; CHECK-NEXT: mov w8, w0668; CHECK-NEXT: // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def $z4_z5_z6_z7669; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3670; CHECK-NEXT: // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def $z4_z5_z6_z7671; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3672; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def $z4_z5_z6_z7673; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3674; CHECK-NEXT: umlal za.s[w8, 0:1, vgx4], { z0.h - z3.h }, { z4.h - z7.h }675; CHECK-NEXT: umlal za.s[w8, 6:7, vgx4], { z0.h - z3.h }, { z4.h - z7.h }676; CHECK-NEXT: ret677 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2, <vscale x 8 x i16> %zm3) {678 call void @llvm.aarch64.sme.umlal.vg2x4.nxv8i16(i32 %slice,679 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,680 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2, <vscale x 8 x i16> %zm3)681 %slice.6 = add i32 %slice, 6682 call void @llvm.aarch64.sme.umlal.vg2x4.nxv8i16(i32 %slice.6,683 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,684 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2, <vscale x 8 x i16> %zm3)685 ret void686}687 688;689; BF/F/S/UMLSL x4 (MULTI)690;691 692define void @multi_vector_sub_multi_vg2x4_bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3,693; CHECK-LABEL: multi_vector_sub_multi_vg2x4_bf16:694; CHECK: // %bb.0:695; CHECK-NEXT: // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def $z4_z5_z6_z7696; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3697; CHECK-NEXT: mov w8, w0698; CHECK-NEXT: // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def $z4_z5_z6_z7699; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3700; CHECK-NEXT: // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def $z4_z5_z6_z7701; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3702; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def $z4_z5_z6_z7703; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3704; CHECK-NEXT: bfmlsl za.s[w8, 0:1, vgx4], { z0.h - z3.h }, { z4.h - z7.h }705; CHECK-NEXT: bfmlsl za.s[w8, 6:7, vgx4], { z0.h - z3.h }, { z4.h - z7.h }706; CHECK-NEXT: ret707 <vscale x 8 x bfloat> %zm0, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2, <vscale x 8 x bfloat> %zm3) {708 call void @llvm.aarch64.sme.fmlsl.vg2x4.nxv8bf16(i32 %slice,709 <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3,710 <vscale x 8 x bfloat> %zm0, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2, <vscale x 8 x bfloat> %zm3)711 %slice.6 = add i32 %slice, 6712 call void @llvm.aarch64.sme.fmlsl.vg2x4.nxv8bf16(i32 %slice.6,713 <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3,714 <vscale x 8 x bfloat> %zm0, <vscale x 8 x bfloat> %zm1, <vscale x 8 x bfloat> %zm2, <vscale x 8 x bfloat> %zm3)715 ret void716}717 718define void @multi_vector_sub_multi_vg2x4_f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3,719; CHECK-LABEL: multi_vector_sub_multi_vg2x4_f16:720; CHECK: // %bb.0:721; CHECK-NEXT: // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def $z4_z5_z6_z7722; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3723; CHECK-NEXT: mov w8, w0724; CHECK-NEXT: // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def $z4_z5_z6_z7725; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3726; CHECK-NEXT: // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def $z4_z5_z6_z7727; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3728; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def $z4_z5_z6_z7729; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3730; CHECK-NEXT: fmlsl za.s[w8, 0:1, vgx4], { z0.h - z3.h }, { z4.h - z7.h }731; CHECK-NEXT: fmlsl za.s[w8, 6:7, vgx4], { z0.h - z3.h }, { z4.h - z7.h }732; CHECK-NEXT: ret733 <vscale x 8 x half> %zm0, <vscale x 8 x half> %zm1, <vscale x 8 x half> %zm2, <vscale x 8 x half> %zm3) {734 call void @llvm.aarch64.sme.fmlsl.vg2x4.nxv8f16(i32 %slice,735 <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3,736 <vscale x 8 x half> %zm0, <vscale x 8 x half> %zm1, <vscale x 8 x half> %zm2, <vscale x 8 x half> %zm3)737 %slice.6 = add i32 %slice, 6738 call void @llvm.aarch64.sme.fmlsl.vg2x4.nxv8f16(i32 %slice.6,739 <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3,740 <vscale x 8 x half> %zm0, <vscale x 8 x half> %zm1, <vscale x 8 x half> %zm2, <vscale x 8 x half> %zm3)741 ret void742}743 744define void @multi_vector_sub_multi_vg2x4_s16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,745; CHECK-LABEL: multi_vector_sub_multi_vg2x4_s16:746; CHECK: // %bb.0:747; CHECK-NEXT: // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def $z4_z5_z6_z7748; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3749; CHECK-NEXT: mov w8, w0750; CHECK-NEXT: // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def $z4_z5_z6_z7751; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3752; CHECK-NEXT: // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def $z4_z5_z6_z7753; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3754; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def $z4_z5_z6_z7755; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3756; CHECK-NEXT: smlsl za.s[w8, 0:1, vgx4], { z0.h - z3.h }, { z4.h - z7.h }757; CHECK-NEXT: smlsl za.s[w8, 6:7, vgx4], { z0.h - z3.h }, { z4.h - z7.h }758; CHECK-NEXT: ret759 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2, <vscale x 8 x i16> %zm3) {760 call void @llvm.aarch64.sme.smlsl.vg2x4.nxv8i16(i32 %slice,761 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,762 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2, <vscale x 8 x i16> %zm3)763 %slice.6 = add i32 %slice, 6764 call void @llvm.aarch64.sme.smlsl.vg2x4.nxv8i16(i32 %slice.6,765 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,766 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2, <vscale x 8 x i16> %zm3)767 ret void768}769 770define void @multi_vector_sub_multi_vg2x4_u16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,771; CHECK-LABEL: multi_vector_sub_multi_vg2x4_u16:772; CHECK: // %bb.0:773; CHECK-NEXT: // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def $z4_z5_z6_z7774; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3775; CHECK-NEXT: mov w8, w0776; CHECK-NEXT: // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def $z4_z5_z6_z7777; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3778; CHECK-NEXT: // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def $z4_z5_z6_z7779; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3780; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def $z4_z5_z6_z7781; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3782; CHECK-NEXT: umlsl za.s[w8, 0:1, vgx4], { z0.h - z3.h }, { z4.h - z7.h }783; CHECK-NEXT: umlsl za.s[w8, 6:7, vgx4], { z0.h - z3.h }, { z4.h - z7.h }784; CHECK-NEXT: ret785 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2, <vscale x 8 x i16> %zm3) {786 call void @llvm.aarch64.sme.umlsl.vg2x4.nxv8i16(i32 %slice,787 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,788 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2, <vscale x 8 x i16> %zm3)789 %slice.6 = add i32 %slice, 6790 call void @llvm.aarch64.sme.umlsl.vg2x4.nxv8i16(i32 %slice.6,791 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,792 <vscale x 8 x i16> %zm0, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2, <vscale x 8 x i16> %zm3)793 ret void794}795 796;797; BF/F/S/UMLAL x1 (INDEXED)798;799 800define void @multi_vector_add_lane_vg2x1_f16(i32 %slice, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm) {801; CHECK-LABEL: multi_vector_add_lane_vg2x1_f16:802; CHECK: // %bb.0:803; CHECK-NEXT: mov w8, w0804; CHECK-NEXT: fmlal za.s[w8, 0:1], z0.h, z1.h[0]805; CHECK-NEXT: fmlal za.s[w8, 14:15], z0.h, z1.h[7]806; CHECK-NEXT: ret807 call void @llvm.aarch64.sme.fmlal.lane.vg2x1.nxv8f16(i32 %slice, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm, i32 0)808 %slice.14 = add i32 %slice, 14809 call void @llvm.aarch64.sme.fmlal.lane.vg2x1.nxv8f16(i32 %slice.14, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm, i32 7)810 ret void811}812 813define void @multi_vector_add_lane_vg2x1_bf16(i32 %slice, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm) {814; CHECK-LABEL: multi_vector_add_lane_vg2x1_bf16:815; CHECK: // %bb.0:816; CHECK-NEXT: mov w8, w0817; CHECK-NEXT: bfmlal za.s[w8, 0:1], z0.h, z1.h[0]818; CHECK-NEXT: bfmlal za.s[w8, 14:15], z0.h, z1.h[7]819; CHECK-NEXT: ret820 call void @llvm.aarch64.sme.fmlal.lane.vg2x1.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm, i32 0)821 %slice.14 = add i32 %slice, 14822 call void @llvm.aarch64.sme.fmlal.lane.vg2x1.nxv8bf16(i32 %slice.14, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm, i32 7)823 ret void824}825 826define void @multi_vector_add_lane_vg2x1_s16(i32 %slice, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm) {827; CHECK-LABEL: multi_vector_add_lane_vg2x1_s16:828; CHECK: // %bb.0:829; CHECK-NEXT: mov w8, w0830; CHECK-NEXT: smlal za.s[w8, 0:1], z0.h, z1.h[0]831; CHECK-NEXT: smlal za.s[w8, 14:15], z0.h, z1.h[7]832; CHECK-NEXT: ret833 call void @llvm.aarch64.sme.smlal.lane.vg2x1.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm, i32 0)834 %slice.14 = add i32 %slice, 14835 call void @llvm.aarch64.sme.smlal.lane.vg2x1.nxv8i16(i32 %slice.14, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm, i32 7)836 ret void837}838 839define void @multi_vector_add_lane_vg2x1_u16(i32 %slice, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm) {840; CHECK-LABEL: multi_vector_add_lane_vg2x1_u16:841; CHECK: // %bb.0:842; CHECK-NEXT: mov w8, w0843; CHECK-NEXT: umlal za.s[w8, 0:1], z0.h, z1.h[0]844; CHECK-NEXT: umlal za.s[w8, 14:15], z0.h, z1.h[7]845; CHECK-NEXT: ret846 call void @llvm.aarch64.sme.umlal.lane.vg2x1.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm, i32 0)847 %slice.14 = add i32 %slice, 14848 call void @llvm.aarch64.sme.umlal.lane.vg2x1.nxv8i16(i32 %slice.14, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm, i32 7)849 ret void850}851 852;853; BF/F/S/UMLSL x1 (INDEXED)854;855 856define void @multi_vector_sub_lane_vg2x1_f16(i32 %slice, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm) {857; CHECK-LABEL: multi_vector_sub_lane_vg2x1_f16:858; CHECK: // %bb.0:859; CHECK-NEXT: mov w8, w0860; CHECK-NEXT: fmlsl za.s[w8, 0:1], z0.h, z1.h[0]861; CHECK-NEXT: fmlsl za.s[w8, 14:15], z0.h, z1.h[7]862; CHECK-NEXT: ret863 call void @llvm.aarch64.sme.fmlsl.lane.vg2x1.nxv8f16(i32 %slice, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm, i32 0)864 %slice.14 = add i32 %slice, 14865 call void @llvm.aarch64.sme.fmlsl.lane.vg2x1.nxv8f16(i32 %slice.14, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm, i32 7)866 ret void867}868 869define void @multi_vector_sub_lane_vg2x1_bf16(i32 %slice, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm) {870; CHECK-LABEL: multi_vector_sub_lane_vg2x1_bf16:871; CHECK: // %bb.0:872; CHECK-NEXT: mov w8, w0873; CHECK-NEXT: bfmlsl za.s[w8, 0:1], z0.h, z1.h[0]874; CHECK-NEXT: bfmlsl za.s[w8, 14:15], z0.h, z1.h[7]875; CHECK-NEXT: ret876 call void @llvm.aarch64.sme.fmlsl.lane.vg2x1.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm, i32 0)877 %slice.14 = add i32 %slice, 14878 call void @llvm.aarch64.sme.fmlsl.lane.vg2x1.nxv8bf16(i32 %slice.14, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm, i32 7)879 ret void880}881 882define void @multi_vector_sub_lane_vg2x1_s16(i32 %slice, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm) {883; CHECK-LABEL: multi_vector_sub_lane_vg2x1_s16:884; CHECK: // %bb.0:885; CHECK-NEXT: mov w8, w0886; CHECK-NEXT: smlsl za.s[w8, 0:1], z0.h, z1.h[0]887; CHECK-NEXT: smlsl za.s[w8, 14:15], z0.h, z1.h[7]888; CHECK-NEXT: ret889 call void @llvm.aarch64.sme.smlsl.lane.vg2x1.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm, i32 0)890 %slice.14 = add i32 %slice, 14891 call void @llvm.aarch64.sme.smlsl.lane.vg2x1.nxv8i16(i32 %slice.14, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm, i32 7)892 ret void893}894 895define void @multi_vector_sub_lane_vg2x1_u16(i32 %slice, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm) {896; CHECK-LABEL: multi_vector_sub_lane_vg2x1_u16:897; CHECK: // %bb.0:898; CHECK-NEXT: mov w8, w0899; CHECK-NEXT: umlsl za.s[w8, 0:1], z0.h, z1.h[0]900; CHECK-NEXT: umlsl za.s[w8, 14:15], z0.h, z1.h[7]901; CHECK-NEXT: ret902 call void @llvm.aarch64.sme.umlsl.lane.vg2x1.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm, i32 0)903 %slice.14 = add i32 %slice, 14904 call void @llvm.aarch64.sme.umlsl.lane.vg2x1.nxv8i16(i32 %slice.14, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm, i32 7)905 ret void906}907 908;909; BF/F/S/UMLAL x2 (INDEXED)910;911 912define void @multi_vector_add_lane_vg2x2_f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zm) {913; CHECK-LABEL: multi_vector_add_lane_vg2x2_f16:914; CHECK: // %bb.0:915; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1916; CHECK-NEXT: mov w8, w0917; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1918; CHECK-NEXT: fmlal za.s[w8, 0:1, vgx2], { z0.h, z1.h }, z2.h[0]919; CHECK-NEXT: fmlal za.s[w8, 6:7, vgx2], { z0.h, z1.h }, z2.h[7]920; CHECK-NEXT: ret921 call void @llvm.aarch64.sme.fmlal.lane.vg2x2.nxv8f16(i32 %slice,922 <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zm, i32 0)923 %slice.6 = add i32 %slice, 6924 call void @llvm.aarch64.sme.fmlal.lane.vg2x2.nxv8f16(i32 %slice.6,925 <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zm, i32 7)926 ret void927}928 929define void @multi_vector_add_lane_vg2x2_bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zm) {930; CHECK-LABEL: multi_vector_add_lane_vg2x2_bf16:931; CHECK: // %bb.0:932; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1933; CHECK-NEXT: mov w8, w0934; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1935; CHECK-NEXT: bfmlal za.s[w8, 0:1, vgx2], { z0.h, z1.h }, z2.h[0]936; CHECK-NEXT: bfmlal za.s[w8, 6:7, vgx2], { z0.h, z1.h }, z2.h[7]937; CHECK-NEXT: ret938 call void @llvm.aarch64.sme.fmlal.lane.vg2x2.nxv8bf16(i32 %slice,939 <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zm, i32 0)940 %slice.6 = add i32 %slice, 6941 call void @llvm.aarch64.sme.fmlal.lane.vg2x2.nxv8bf16(i32 %slice.6,942 <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zm, i32 7)943 ret void944}945 946define void @multi_vector_add_lane_vg2x2_s16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm) {947; CHECK-LABEL: multi_vector_add_lane_vg2x2_s16:948; CHECK: // %bb.0:949; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1950; CHECK-NEXT: mov w8, w0951; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1952; CHECK-NEXT: smlal za.s[w8, 0:1, vgx2], { z0.h, z1.h }, z2.h[0]953; CHECK-NEXT: smlal za.s[w8, 6:7, vgx2], { z0.h, z1.h }, z2.h[7]954; CHECK-NEXT: ret955 call void @llvm.aarch64.sme.smlal.lane.vg2x2.nxv8i16(i32 %slice,956 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm, i32 0)957 %slice.6 = add i32 %slice, 6958 call void @llvm.aarch64.sme.smlal.lane.vg2x2.nxv8i16(i32 %slice.6,959 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm, i32 7)960 ret void961}962 963define void @multi_vector_add_lane_vg2x2_u16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm) {964; CHECK-LABEL: multi_vector_add_lane_vg2x2_u16:965; CHECK: // %bb.0:966; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1967; CHECK-NEXT: mov w8, w0968; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1969; CHECK-NEXT: umlal za.s[w8, 0:1, vgx2], { z0.h, z1.h }, z2.h[0]970; CHECK-NEXT: umlal za.s[w8, 6:7, vgx2], { z0.h, z1.h }, z2.h[7]971; CHECK-NEXT: ret972 call void @llvm.aarch64.sme.umlal.lane.vg2x2.nxv8i16(i32 %slice,973 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm, i32 0)974 %slice.6 = add i32 %slice, 6975 call void @llvm.aarch64.sme.umlal.lane.vg2x2.nxv8i16(i32 %slice.6,976 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm, i32 7)977 ret void978}979 980;981; BF/F/S/UMLSL x2 (INDEXED)982;983 984define void @multi_vector_sub_lane_vg2x2_f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zm) {985; CHECK-LABEL: multi_vector_sub_lane_vg2x2_f16:986; CHECK: // %bb.0:987; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1988; CHECK-NEXT: mov w8, w0989; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1990; CHECK-NEXT: fmlsl za.s[w8, 0:1, vgx2], { z0.h, z1.h }, z2.h[0]991; CHECK-NEXT: fmlsl za.s[w8, 6:7, vgx2], { z0.h, z1.h }, z2.h[7]992; CHECK-NEXT: ret993 call void @llvm.aarch64.sme.fmlsl.lane.vg2x2.nxv8f16(i32 %slice,994 <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zm, i32 0)995 %slice.6 = add i32 %slice, 6996 call void @llvm.aarch64.sme.fmlsl.lane.vg2x2.nxv8f16(i32 %slice.6,997 <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zm, i32 7)998 ret void999}1000 1001define void @multi_vector_sub_lane_vg2x2_bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zm) {1002; CHECK-LABEL: multi_vector_sub_lane_vg2x2_bf16:1003; CHECK: // %bb.0:1004; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z11005; CHECK-NEXT: mov w8, w01006; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z11007; CHECK-NEXT: bfmlsl za.s[w8, 0:1, vgx2], { z0.h, z1.h }, z2.h[0]1008; CHECK-NEXT: bfmlsl za.s[w8, 6:7, vgx2], { z0.h, z1.h }, z2.h[7]1009; CHECK-NEXT: ret1010 call void @llvm.aarch64.sme.fmlsl.lane.vg2x2.nxv8bf16(i32 %slice,1011 <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zm, i32 0)1012 %slice.6 = add i32 %slice, 61013 call void @llvm.aarch64.sme.fmlsl.lane.vg2x2.nxv8bf16(i32 %slice.6,1014 <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zm, i32 7)1015 ret void1016}1017 1018define void @multi_vector_sub_lane_vg2x2_s16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm) {1019; CHECK-LABEL: multi_vector_sub_lane_vg2x2_s16:1020; CHECK: // %bb.0:1021; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z11022; CHECK-NEXT: mov w8, w01023; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z11024; CHECK-NEXT: smlsl za.s[w8, 0:1, vgx2], { z0.h, z1.h }, z2.h[0]1025; CHECK-NEXT: smlsl za.s[w8, 6:7, vgx2], { z0.h, z1.h }, z2.h[7]1026; CHECK-NEXT: ret1027 call void @llvm.aarch64.sme.smlsl.lane.vg2x2.nxv8i16(i32 %slice,1028 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm, i32 0)1029 %slice.6 = add i32 %slice, 61030 call void @llvm.aarch64.sme.smlsl.lane.vg2x2.nxv8i16(i32 %slice.6,1031 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm, i32 7)1032 ret void1033}1034 1035define void @multi_vector_sub_lane_vg2x2_u16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm) {1036; CHECK-LABEL: multi_vector_sub_lane_vg2x2_u16:1037; CHECK: // %bb.0:1038; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z11039; CHECK-NEXT: mov w8, w01040; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z11041; CHECK-NEXT: umlsl za.s[w8, 0:1, vgx2], { z0.h, z1.h }, z2.h[0]1042; CHECK-NEXT: umlsl za.s[w8, 6:7, vgx2], { z0.h, z1.h }, z2.h[7]1043; CHECK-NEXT: ret1044 call void @llvm.aarch64.sme.umlsl.lane.vg2x2.nxv8i16(i32 %slice,1045 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm, i32 0)1046 %slice.6 = add i32 %slice, 61047 call void @llvm.aarch64.sme.umlsl.lane.vg2x2.nxv8i16(i32 %slice.6,1048 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zm, i32 7)1049 ret void1050}1051 1052;1053; BF/F/S/UMLAL x4 (INDEXED)1054;1055 1056define void @multi_vector_add_lane_vg2x4_f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3, <vscale x 8 x half> %zm) {1057; CHECK-LABEL: multi_vector_add_lane_vg2x4_f16:1058; CHECK: // %bb.0:1059; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z31060; CHECK-NEXT: mov w8, w01061; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z31062; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z31063; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z31064; CHECK-NEXT: fmlal za.s[w8, 0:1, vgx4], { z0.h - z3.h }, z4.h[0]1065; CHECK-NEXT: fmlal za.s[w8, 6:7, vgx4], { z0.h - z3.h }, z4.h[7]1066; CHECK-NEXT: ret1067 call void @llvm.aarch64.sme.fmlal.lane.vg2x4.nxv8f16(i32 %slice,1068 <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3,1069 <vscale x 8 x half> %zm, i32 0)1070 %slice.6 = add i32 %slice, 61071 call void @llvm.aarch64.sme.fmlal.lane.vg2x4.nxv8f16(i32 %slice.6,1072 <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3,1073 <vscale x 8 x half> %zm, i32 7)1074 ret void1075}1076 1077define void @multi_vector_add_lane_vg2x4_bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3, <vscale x 8 x bfloat> %zm) {1078; CHECK-LABEL: multi_vector_add_lane_vg2x4_bf16:1079; CHECK: // %bb.0:1080; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z31081; CHECK-NEXT: mov w8, w01082; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z31083; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z31084; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z31085; CHECK-NEXT: bfmlal za.s[w8, 0:1, vgx4], { z0.h - z3.h }, z4.h[0]1086; CHECK-NEXT: bfmlal za.s[w8, 6:7, vgx4], { z0.h - z3.h }, z4.h[7]1087; CHECK-NEXT: ret1088 call void @llvm.aarch64.sme.fmlal.lane.vg2x4.nxv8bf16(i32 %slice,1089 <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3,1090 <vscale x 8 x bfloat> %zm, i32 0)1091 %slice.6 = add i32 %slice, 61092 call void @llvm.aarch64.sme.fmlal.lane.vg2x4.nxv8bf16(i32 %slice.6,1093 <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3,1094 <vscale x 8 x bfloat> %zm, i32 7)1095 ret void1096}1097 1098define void @multi_vector_add_lane_vg2x4_s16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zm) {1099; CHECK-LABEL: multi_vector_add_lane_vg2x4_s16:1100; CHECK: // %bb.0:1101; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z31102; CHECK-NEXT: mov w8, w01103; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z31104; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z31105; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z31106; CHECK-NEXT: smlal za.s[w8, 0:1, vgx4], { z0.h - z3.h }, z4.h[0]1107; CHECK-NEXT: smlal za.s[w8, 6:7, vgx4], { z0.h - z3.h }, z4.h[7]1108; CHECK-NEXT: ret1109 call void @llvm.aarch64.sme.smlal.lane.vg2x4.nxv8i16(i32 %slice,1110 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,1111 <vscale x 8 x i16> %zm, i32 0)1112 %slice.6 = add i32 %slice, 61113 call void @llvm.aarch64.sme.smlal.lane.vg2x4.nxv8i16(i32 %slice.6,1114 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,1115 <vscale x 8 x i16> %zm, i32 7)1116 ret void1117}1118 1119define void @multi_vector_add_lane_vg2x4_u16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zm) {1120; CHECK-LABEL: multi_vector_add_lane_vg2x4_u16:1121; CHECK: // %bb.0:1122; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z31123; CHECK-NEXT: mov w8, w01124; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z31125; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z31126; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z31127; CHECK-NEXT: umlal za.s[w8, 0:1, vgx4], { z0.h - z3.h }, z4.h[0]1128; CHECK-NEXT: umlal za.s[w8, 6:7, vgx4], { z0.h - z3.h }, z4.h[7]1129; CHECK-NEXT: ret1130 call void @llvm.aarch64.sme.umlal.lane.vg2x4.nxv8i16(i32 %slice,1131 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,1132 <vscale x 8 x i16> %zm, i32 0)1133 %slice.6 = add i32 %slice, 61134 call void @llvm.aarch64.sme.umlal.lane.vg2x4.nxv8i16(i32 %slice.6,1135 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,1136 <vscale x 8 x i16> %zm, i32 7)1137 ret void1138}1139 1140;1141; BF/F/S/UMLSL x4 (INDEXED)1142;1143 1144define void @multi_vector_sub_lane_vg2x4_f16(i32 %slice, <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3, <vscale x 8 x half> %zm) {1145; CHECK-LABEL: multi_vector_sub_lane_vg2x4_f16:1146; CHECK: // %bb.0:1147; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z31148; CHECK-NEXT: mov w8, w01149; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z31150; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z31151; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z31152; CHECK-NEXT: fmlsl za.s[w8, 0:1, vgx4], { z0.h - z3.h }, z4.h[0]1153; CHECK-NEXT: fmlsl za.s[w8, 6:7, vgx4], { z0.h - z3.h }, z4.h[7]1154; CHECK-NEXT: ret1155 call void @llvm.aarch64.sme.fmlsl.lane.vg2x4.nxv8f16(i32 %slice,1156 <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3,1157 <vscale x 8 x half> %zm, i32 0)1158 %slice.6 = add i32 %slice, 61159 call void @llvm.aarch64.sme.fmlsl.lane.vg2x4.nxv8f16(i32 %slice.6,1160 <vscale x 8 x half> %zn0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3,1161 <vscale x 8 x half> %zm, i32 7)1162 ret void1163}1164 1165define void @multi_vector_sub_lane_vg2x4_bf16(i32 %slice, <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3, <vscale x 8 x bfloat> %zm) {1166; CHECK-LABEL: multi_vector_sub_lane_vg2x4_bf16:1167; CHECK: // %bb.0:1168; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z31169; CHECK-NEXT: mov w8, w01170; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z31171; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z31172; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z31173; CHECK-NEXT: bfmlsl za.s[w8, 0:1, vgx4], { z0.h - z3.h }, z4.h[0]1174; CHECK-NEXT: bfmlsl za.s[w8, 6:7, vgx4], { z0.h - z3.h }, z4.h[7]1175; CHECK-NEXT: ret1176 call void @llvm.aarch64.sme.fmlsl.lane.vg2x4.nxv8bf16(i32 %slice,1177 <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3,1178 <vscale x 8 x bfloat> %zm, i32 0)1179 %slice.6 = add i32 %slice, 61180 call void @llvm.aarch64.sme.fmlsl.lane.vg2x4.nxv8bf16(i32 %slice.6,1181 <vscale x 8 x bfloat> %zn0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3,1182 <vscale x 8 x bfloat> %zm, i32 7)1183 ret void1184}1185 1186define void @multi_vector_sub_lane_vg2x4_s16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zm) {1187; CHECK-LABEL: multi_vector_sub_lane_vg2x4_s16:1188; CHECK: // %bb.0:1189; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z31190; CHECK-NEXT: mov w8, w01191; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z31192; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z31193; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z31194; CHECK-NEXT: smlsl za.s[w8, 0:1, vgx4], { z0.h - z3.h }, z4.h[0]1195; CHECK-NEXT: smlsl za.s[w8, 6:7, vgx4], { z0.h - z3.h }, z4.h[7]1196; CHECK-NEXT: ret1197 call void @llvm.aarch64.sme.smlsl.lane.vg2x4.nxv8i16(i32 %slice,1198 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,1199 <vscale x 8 x i16> %zm, i32 0)1200 %slice.6 = add i32 %slice, 61201 call void @llvm.aarch64.sme.smlsl.lane.vg2x4.nxv8i16(i32 %slice.6,1202 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,1203 <vscale x 8 x i16> %zm, i32 7)1204 ret void1205}1206 1207define void @multi_vector_sub_lane_vg2x4_u16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zm) {1208; CHECK-LABEL: multi_vector_sub_lane_vg2x4_u16:1209; CHECK: // %bb.0:1210; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z31211; CHECK-NEXT: mov w8, w01212; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z31213; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z31214; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z31215; CHECK-NEXT: umlsl za.s[w8, 0:1, vgx4], { z0.h - z3.h }, z4.h[0]1216; CHECK-NEXT: umlsl za.s[w8, 6:7, vgx4], { z0.h - z3.h }, z4.h[7]1217; CHECK-NEXT: ret1218 call void @llvm.aarch64.sme.umlsl.lane.vg2x4.nxv8i16(i32 %slice,1219 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,1220 <vscale x 8 x i16> %zm, i32 0)1221 %slice.6 = add i32 %slice, 61222 call void @llvm.aarch64.sme.umlsl.lane.vg2x4.nxv8i16(i32 %slice.6,1223 <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,1224 <vscale x 8 x i16> %zm, i32 7)1225 ret void1226}1227 1228declare void @llvm.aarch64.sme.fmlal.single.vg2x1.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)1229declare void @llvm.aarch64.sme.fmlal.single.vg2x1.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>)1230declare void @llvm.aarch64.sme.smlal.single.vg2x1.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>)1231declare void @llvm.aarch64.sme.umlal.single.vg2x1.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>)1232 1233declare void @llvm.aarch64.sme.fmlsl.single.vg2x1.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)1234declare void @llvm.aarch64.sme.fmlsl.single.vg2x1.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>)1235declare void @llvm.aarch64.sme.smlsl.single.vg2x1.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>)1236declare void @llvm.aarch64.sme.umlsl.single.vg2x1.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>)1237 1238declare void @llvm.aarch64.sme.fmlal.single.vg2x2.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)1239declare void @llvm.aarch64.sme.fmlal.single.vg2x2.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)1240declare void @llvm.aarch64.sme.smlal.single.vg2x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)1241declare void @llvm.aarch64.sme.umlal.single.vg2x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)1242 1243declare void @llvm.aarch64.sme.fmlsl.single.vg2x2.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)1244declare void @llvm.aarch64.sme.fmlsl.single.vg2x2.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)1245declare void @llvm.aarch64.sme.smlsl.single.vg2x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)1246declare void @llvm.aarch64.sme.umlsl.single.vg2x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)1247 1248declare void @llvm.aarch64.sme.fmlal.single.vg2x4.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>,1249 <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)1250declare void @llvm.aarch64.sme.fmlal.single.vg2x4.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>,1251 <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)1252declare void @llvm.aarch64.sme.smlal.single.vg2x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>,1253 <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)1254declare void @llvm.aarch64.sme.umlal.single.vg2x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>,1255 <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)1256 1257declare void @llvm.aarch64.sme.fmlsl.single.vg2x4.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>,1258 <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)1259declare void @llvm.aarch64.sme.fmlsl.single.vg2x4.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>,1260 <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)1261declare void @llvm.aarch64.sme.smlsl.single.vg2x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>,1262 <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)1263declare void @llvm.aarch64.sme.umlsl.single.vg2x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>,1264 <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)1265 1266declare void @llvm.aarch64.sme.fmlal.vg2x2.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)1267declare void @llvm.aarch64.sme.fmlal.vg2x2.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)1268declare void @llvm.aarch64.sme.smlal.vg2x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)1269declare void @llvm.aarch64.sme.umlal.vg2x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)1270 1271declare void @llvm.aarch64.sme.fmlsl.vg2x2.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)1272declare void @llvm.aarch64.sme.fmlsl.vg2x2.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)1273declare void @llvm.aarch64.sme.smlsl.vg2x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)1274declare void @llvm.aarch64.sme.umlsl.vg2x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)1275 1276declare void @llvm.aarch64.sme.fmlal.vg2x4.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>,1277 <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)1278declare void @llvm.aarch64.sme.fmlal.vg2x4.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>,1279 <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)1280declare void @llvm.aarch64.sme.smlal.vg2x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>,1281 <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)1282declare void @llvm.aarch64.sme.umlal.vg2x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>,1283 <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)1284 1285declare void @llvm.aarch64.sme.fmlsl.vg2x4.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>,1286 <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)1287declare void @llvm.aarch64.sme.fmlsl.vg2x4.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>,1288 <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)1289declare void @llvm.aarch64.sme.smlsl.vg2x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>,1290 <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)1291declare void @llvm.aarch64.sme.umlsl.vg2x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>,1292 <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)1293 1294declare void @llvm.aarch64.sme.fmlal.lane.vg2x1.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, i32)1295declare void @llvm.aarch64.sme.fmlal.lane.vg2x1.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>, i32)1296declare void @llvm.aarch64.sme.smlal.lane.vg2x1.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)1297declare void @llvm.aarch64.sme.umlal.lane.vg2x1.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)1298 1299declare void @llvm.aarch64.sme.fmlsl.lane.vg2x1.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, i32)1300declare void @llvm.aarch64.sme.fmlsl.lane.vg2x1.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>, i32)1301declare void @llvm.aarch64.sme.smlsl.lane.vg2x1.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)1302declare void @llvm.aarch64.sme.umlsl.lane.vg2x1.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)1303 1304declare void @llvm.aarch64.sme.fmlal.lane.vg2x2.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, i32)1305declare void @llvm.aarch64.sme.fmlal.lane.vg2x2.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, i32)1306declare void @llvm.aarch64.sme.smlal.lane.vg2x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)1307declare void @llvm.aarch64.sme.umlal.lane.vg2x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)1308 1309declare void @llvm.aarch64.sme.fmlsl.lane.vg2x2.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, i32)1310declare void @llvm.aarch64.sme.fmlsl.lane.vg2x2.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, i32)1311declare void @llvm.aarch64.sme.smlsl.lane.vg2x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)1312declare void @llvm.aarch64.sme.umlsl.lane.vg2x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)1313 1314declare void @llvm.aarch64.sme.fmlal.lane.vg2x4.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, i32)1315declare void @llvm.aarch64.sme.fmlal.lane.vg2x4.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, i32)1316declare void @llvm.aarch64.sme.smlal.lane.vg2x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)1317declare void @llvm.aarch64.sme.umlal.lane.vg2x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)1318 1319declare void @llvm.aarch64.sme.fmlsl.lane.vg2x4.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, i32)1320declare void @llvm.aarch64.sme.fmlsl.lane.vg2x4.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, i32)1321declare void @llvm.aarch64.sme.smlsl.lane.vg2x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)1322declare void @llvm.aarch64.sme.umlsl.lane.vg2x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)1323