821 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming -enable-subreg-liveness -verify-machineinstrs < %s | FileCheck %s3 4;5; Move Multi-Vector To Tile (Write) x 26;7 8; Horizontal9 10define void @za_write_vg2_horiz_b(i32 %slice, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2) {11; CHECK-LABEL: za_write_vg2_horiz_b:12; CHECK: // %bb.0:13; CHECK-NEXT: mov w12, w014; CHECK-NEXT: mov za0h.b[w12, 0:1], { z0.b, z1.b }15; CHECK-NEXT: mov za0h.b[w12, 14:15], { z0.b, z1.b }16; CHECK-NEXT: ret17 call void @llvm.aarch64.sme.write.hor.vg2.nxv16i8(i32 0, i32 %slice, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2)18 %slice.14 = add i32 %slice, 1419 call void @llvm.aarch64.sme.write.hor.vg2.nxv16i8(i32 0, i32 %slice.14, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2)20 ret void21}22 23define void @za_write_vg2_horiz_b_tuple(i64 %stride, ptr %ptr) {24; CHECK-LABEL: za_write_vg2_horiz_b_tuple:25; CHECK: // %bb.0: // %entry26; CHECK-NEXT: ptrue pn8.b27; CHECK-NEXT: mov w12, wzr28; CHECK-NEXT: ld1b { z16.b, z24.b }, pn8/z, [x1]29; CHECK-NEXT: ld1b { z17.b, z25.b }, pn8/z, [x1, x0]30; CHECK-NEXT: mov za0h.b[w12, 0:1], { z16.b, z17.b }31; CHECK-NEXT: mov za0h.b[w12, 0:1], { z24.b, z25.b }32; CHECK-NEXT: ret33entry:34 %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()35 %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)36 %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 037 %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 138 %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride39 %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)40 %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 041 %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 142 call void @llvm.aarch64.sme.write.hor.vg2.nxv16i8(i32 0, i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5)43 call void @llvm.aarch64.sme.write.hor.vg2.nxv16i8(i32 0, i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6)44 ret void45}46 47define void @za_write_vg2_horiz_h(i32 %slice, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2) {48; CHECK-LABEL: za_write_vg2_horiz_h:49; CHECK: // %bb.0:50; CHECK-NEXT: mov w12, w051; CHECK-NEXT: mov za0h.h[w12, 0:1], { z0.h, z1.h }52; CHECK-NEXT: mov za1h.h[w12, 6:7], { z0.h, z1.h }53; CHECK-NEXT: ret54 call void @llvm.aarch64.sme.write.hor.vg2.nxv8i16(i32 0, i32 %slice, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2)55 %slice.6 = add i32 %slice, 656 call void @llvm.aarch64.sme.write.hor.vg2.nxv8i16(i32 1, i32 %slice.6, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2)57 ret void58}59 60define void @za_write_vg2_horiz_f16(i32 %slice, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2) {61; CHECK-LABEL: za_write_vg2_horiz_f16:62; CHECK: // %bb.0:63; CHECK-NEXT: mov w12, w064; CHECK-NEXT: mov za0h.h[w12, 0:1], { z0.h, z1.h }65; CHECK-NEXT: mov za1h.h[w12, 6:7], { z0.h, z1.h }66; CHECK-NEXT: ret67 call void @llvm.aarch64.sme.write.hor.vg2.nxv8f16(i32 0, i32 %slice, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2)68 %slice.6 = add i32 %slice, 669 call void @llvm.aarch64.sme.write.hor.vg2.nxv8f16(i32 1, i32 %slice.6, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2)70 ret void71}72 73define void @za_write_vg2_horiz_bf16(i32 %slice, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2) {74; CHECK-LABEL: za_write_vg2_horiz_bf16:75; CHECK: // %bb.0:76; CHECK-NEXT: mov w12, w077; CHECK-NEXT: mov za0h.h[w12, 0:1], { z0.h, z1.h }78; CHECK-NEXT: mov za1h.h[w12, 6:7], { z0.h, z1.h }79; CHECK-NEXT: ret80 call void @llvm.aarch64.sme.write.hor.vg2.nxv8bf16(i32 0, i32 %slice, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2)81 %slice.6 = add i32 %slice, 682 call void @llvm.aarch64.sme.write.hor.vg2.nxv8bf16(i32 1, i32 %slice.6, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2)83 ret void84}85 86define void @za_write_vg2_horiz_s(i32 %slice, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2) {87; CHECK-LABEL: za_write_vg2_horiz_s:88; CHECK: // %bb.0:89; CHECK-NEXT: mov w12, w090; CHECK-NEXT: mov za0h.s[w12, 0:1], { z0.s, z1.s }91; CHECK-NEXT: mov za3h.s[w12, 2:3], { z0.s, z1.s }92; CHECK-NEXT: ret93 call void @llvm.aarch64.sme.write.hor.vg2.nxv4i32(i32 0, i32 %slice, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2)94 %slice.2 = add i32 %slice, 295 call void @llvm.aarch64.sme.write.hor.vg2.nxv4i32(i32 3, i32 %slice.2, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2)96 ret void97}98 99define void @za_write_vg2_horiz_f32(i32 %slice, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2) {100; CHECK-LABEL: za_write_vg2_horiz_f32:101; CHECK: // %bb.0:102; CHECK-NEXT: mov w12, w0103; CHECK-NEXT: mov za0h.s[w12, 0:1], { z0.s, z1.s }104; CHECK-NEXT: mov za3h.s[w12, 2:3], { z0.s, z1.s }105; CHECK-NEXT: ret106 call void @llvm.aarch64.sme.write.hor.vg2.nxv4f32(i32 0, i32 %slice, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2)107 %slice.2 = add i32 %slice, 2108 call void @llvm.aarch64.sme.write.hor.vg2.nxv4f32(i32 3, i32 %slice.2, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2)109 ret void110}111 112define void @za_write_vg2_horiz_d(i32 %slice, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2) {113; CHECK-LABEL: za_write_vg2_horiz_d:114; CHECK: // %bb.0:115; CHECK-NEXT: mov w12, w0116; CHECK-NEXT: mov za0h.d[w12, 0:1], { z0.d, z1.d }117; CHECK-NEXT: ret118 call void @llvm.aarch64.sme.write.hor.vg2.nxv2i64(i32 0, i32 %slice, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2)119 ret void120}121 122define void @za_write_vg2_horiz_f64(i32 %slice, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2) {123; CHECK-LABEL: za_write_vg2_horiz_f64:124; CHECK: // %bb.0:125; CHECK-NEXT: mov w12, w0126; CHECK-NEXT: mov za0h.d[w12, 0:1], { z0.d, z1.d }127; CHECK-NEXT: ret128 call void @llvm.aarch64.sme.write.hor.vg2.nxv2f64(i32 0, i32 %slice, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2)129 ret void130}131 132; Vertical133 134define void @za_write_vg2_vert_b(i32 %slice, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2) {135; CHECK-LABEL: za_write_vg2_vert_b:136; CHECK: // %bb.0:137; CHECK-NEXT: mov w12, w0138; CHECK-NEXT: mov za0v.b[w12, 0:1], { z0.b, z1.b }139; CHECK-NEXT: mov za0v.b[w12, 14:15], { z0.b, z1.b }140; CHECK-NEXT: ret141 call void @llvm.aarch64.sme.write.ver.vg2.nxv16i8(i32 0, i32 %slice, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2)142 %slice.14 = add i32 %slice, 14143 call void @llvm.aarch64.sme.write.ver.vg2.nxv16i8(i32 0, i32 %slice.14, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2)144 ret void145}146 147define void @za_write_vg2_vert_h(i32 %slice, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2) {148; CHECK-LABEL: za_write_vg2_vert_h:149; CHECK: // %bb.0:150; CHECK-NEXT: mov w12, w0151; CHECK-NEXT: mov za0v.h[w12, 0:1], { z0.h, z1.h }152; CHECK-NEXT: mov za1v.h[w12, 6:7], { z0.h, z1.h }153; CHECK-NEXT: ret154 call void @llvm.aarch64.sme.write.ver.vg2.nxv8i16(i32 0, i32 %slice, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2)155 %slice.6 = add i32 %slice, 6156 call void @llvm.aarch64.sme.write.ver.vg2.nxv8i16(i32 1, i32 %slice.6, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2)157 ret void158}159 160define void @za_write_vg2_vert_f16(i32 %slice, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2) {161; CHECK-LABEL: za_write_vg2_vert_f16:162; CHECK: // %bb.0:163; CHECK-NEXT: mov w12, w0164; CHECK-NEXT: mov za0v.h[w12, 0:1], { z0.h, z1.h }165; CHECK-NEXT: mov za1v.h[w12, 6:7], { z0.h, z1.h }166; CHECK-NEXT: ret167 call void @llvm.aarch64.sme.write.ver.vg2.nxv8f16(i32 0, i32 %slice, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2)168 %slice.6 = add i32 %slice, 6169 call void @llvm.aarch64.sme.write.ver.vg2.nxv8f16(i32 1, i32 %slice.6, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2)170 ret void171}172 173define void @za_write_vg2_vert_bf16(i32 %slice, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2) {174; CHECK-LABEL: za_write_vg2_vert_bf16:175; CHECK: // %bb.0:176; CHECK-NEXT: mov w12, w0177; CHECK-NEXT: mov za0v.h[w12, 0:1], { z0.h, z1.h }178; CHECK-NEXT: mov za1v.h[w12, 6:7], { z0.h, z1.h }179; CHECK-NEXT: ret180 call void @llvm.aarch64.sme.write.ver.vg2.nxv8bf16(i32 0, i32 %slice, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2)181 %slice.6 = add i32 %slice, 6182 call void @llvm.aarch64.sme.write.ver.vg2.nxv8bf16(i32 1, i32 %slice.6, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2)183 ret void184}185 186define void @za_write_vg2_vert_s(i32 %slice, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2) {187; CHECK-LABEL: za_write_vg2_vert_s:188; CHECK: // %bb.0:189; CHECK-NEXT: mov w12, w0190; CHECK-NEXT: mov za0v.s[w12, 0:1], { z0.s, z1.s }191; CHECK-NEXT: mov za3v.s[w12, 2:3], { z0.s, z1.s }192; CHECK-NEXT: ret193 call void @llvm.aarch64.sme.write.ver.vg2.nxv4i32(i32 0, i32 %slice, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2)194 %slice.2 = add i32 %slice, 2195 call void @llvm.aarch64.sme.write.ver.vg2.nxv4i32(i32 3, i32 %slice.2, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2)196 ret void197}198 199define void @za_write_vg2_vert_f32(i32 %slice, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2) {200; CHECK-LABEL: za_write_vg2_vert_f32:201; CHECK: // %bb.0:202; CHECK-NEXT: mov w12, w0203; CHECK-NEXT: mov za0v.s[w12, 0:1], { z0.s, z1.s }204; CHECK-NEXT: mov za3v.s[w12, 2:3], { z0.s, z1.s }205; CHECK-NEXT: ret206 call void @llvm.aarch64.sme.write.ver.vg2.nxv4f32(i32 0, i32 %slice, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2)207 %slice.2 = add i32 %slice, 2208 call void @llvm.aarch64.sme.write.ver.vg2.nxv4f32(i32 3, i32 %slice.2, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2)209 ret void210}211 212define void @za_write_vg2_vert_d(i32 %slice, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2) {213; CHECK-LABEL: za_write_vg2_vert_d:214; CHECK: // %bb.0:215; CHECK-NEXT: mov w12, w0216; CHECK-NEXT: mov za0v.d[w12, 0:1], { z0.d, z1.d }217; CHECK-NEXT: ret218 call void @llvm.aarch64.sme.write.ver.vg2.nxv2i64(i32 0, i32 %slice, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2)219 ret void220}221 222define void @za_write_vg2_vert_f64(i32 %slice, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2) {223; CHECK-LABEL: za_write_vg2_vert_f64:224; CHECK: // %bb.0:225; CHECK-NEXT: mov w12, w0226; CHECK-NEXT: mov za0v.d[w12, 0:1], { z0.d, z1.d }227; CHECK-NEXT: ret228 call void @llvm.aarch64.sme.write.ver.vg2.nxv2f64(i32 0, i32 %slice, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2)229 ret void230}231 232;233; Move Multi-Vector To Tile (Write) x 4234;235 236; Horizontal237 238define void @za_write_vg4_horiz_b(i32 %slice, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4) {239; CHECK-LABEL: za_write_vg4_horiz_b:240; CHECK: // %bb.0:241; CHECK-NEXT: mov w12, w0242; CHECK-NEXT: mov za0h.b[w12, 0:3], { z0.b - z3.b }243; CHECK-NEXT: mov za0h.b[w12, 12:15], { z0.b - z3.b }244; CHECK-NEXT: ret245 call void @llvm.aarch64.sme.write.hor.vg4.nxv16i8(i32 0, i32 %slice, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4)246 %slice.12 = add i32 %slice, 12247 call void @llvm.aarch64.sme.write.hor.vg4.nxv16i8(i32 0, i32 %slice.12, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4)248 ret void249}250 251define void @za_write_vg4_horiz_h(i32 %slice, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4) {252; CHECK-LABEL: za_write_vg4_horiz_h:253; CHECK: // %bb.0:254; CHECK-NEXT: mov w12, w0255; CHECK-NEXT: mov za0h.h[w12, 0:3], { z0.h - z3.h }256; CHECK-NEXT: mov za1h.h[w12, 4:7], { z0.h - z3.h }257; CHECK-NEXT: ret258 call void @llvm.aarch64.sme.write.hor.vg4.nxv8i16(i32 0, i32 %slice, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4)259 %slice.4 = add i32 %slice, 4260 call void @llvm.aarch64.sme.write.hor.vg4.nxv8i16(i32 1, i32 %slice.4, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4)261 ret void262}263 264define void @za_write_vg4_horiz_f16(i32 %slice, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3, <vscale x 8 x half> %zn4) {265; CHECK-LABEL: za_write_vg4_horiz_f16:266; CHECK: // %bb.0:267; CHECK-NEXT: mov w12, w0268; CHECK-NEXT: mov za0h.h[w12, 0:3], { z0.h - z3.h }269; CHECK-NEXT: mov za1h.h[w12, 4:7], { z0.h - z3.h }270; CHECK-NEXT: ret271 call void @llvm.aarch64.sme.write.hor.vg4.nxv8f16(i32 0, i32 %slice, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3, <vscale x 8 x half> %zn4)272 %slice.4 = add i32 %slice, 4273 call void @llvm.aarch64.sme.write.hor.vg4.nxv8f16(i32 1, i32 %slice.4, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3, <vscale x 8 x half> %zn4)274 ret void275}276 277define void @za_write_vg4_horiz_bf16(i32 %slice, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3, <vscale x 8 x bfloat> %zn4) {278; CHECK-LABEL: za_write_vg4_horiz_bf16:279; CHECK: // %bb.0:280; CHECK-NEXT: mov w12, w0281; CHECK-NEXT: mov za0h.h[w12, 0:3], { z0.h - z3.h }282; CHECK-NEXT: mov za1h.h[w12, 4:7], { z0.h - z3.h }283; CHECK-NEXT: ret284 call void @llvm.aarch64.sme.write.hor.vg4.nxv8bf16(i32 0, i32 %slice, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3, <vscale x 8 x bfloat> %zn4)285 %slice.4 = add i32 %slice, 4286 call void @llvm.aarch64.sme.write.hor.vg4.nxv8bf16(i32 1, i32 %slice.4, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3, <vscale x 8 x bfloat> %zn4)287 ret void288}289 290define void @za_write_vg4_horiz_s(i32 %slice, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4) {291; CHECK-LABEL: za_write_vg4_horiz_s:292; CHECK: // %bb.0:293; CHECK-NEXT: mov w12, w0294; CHECK-NEXT: mov za0h.s[w12, 0:3], { z0.s - z3.s }295; CHECK-NEXT: ret296 call void @llvm.aarch64.sme.write.hor.vg4.nxv4i32(i32 0, i32 %slice, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4)297 ret void298}299 300define void @za_write_vg4_horiz_f32(i32 %slice, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3, <vscale x 4 x float> %zn4) {301; CHECK-LABEL: za_write_vg4_horiz_f32:302; CHECK: // %bb.0:303; CHECK-NEXT: mov w12, w0304; CHECK-NEXT: mov za0h.s[w12, 0:3], { z0.s - z3.s }305; CHECK-NEXT: ret306 call void @llvm.aarch64.sme.write.hor.vg4.nxv4f32(i32 0, i32 %slice, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3, <vscale x 4 x float> %zn4)307 ret void308}309 310define void @za_write_vg4_horiz_d(i32 %slice, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4) {311; CHECK-LABEL: za_write_vg4_horiz_d:312; CHECK: // %bb.0:313; CHECK-NEXT: mov w12, w0314; CHECK-NEXT: mov za0h.d[w12, 0:3], { z0.d - z3.d }315; CHECK-NEXT: ret316 call void @llvm.aarch64.sme.write.hor.vg4.nxv2i64(i32 0, i32 %slice, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4)317 ret void318}319 320define void @za_write_vg4_horiz_f64(i32 %slice, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3, <vscale x 2 x double> %zn4) {321; CHECK-LABEL: za_write_vg4_horiz_f64:322; CHECK: // %bb.0:323; CHECK-NEXT: mov w12, w0324; CHECK-NEXT: mov za0h.d[w12, 0:3], { z0.d - z3.d }325; CHECK-NEXT: ret326 call void @llvm.aarch64.sme.write.hor.vg4.nxv2f64(i32 0, i32 %slice, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3, <vscale x 2 x double> %zn4)327 ret void328}329 330; Vertical331 332define void @za_write_vg4_vert_b(i32 %slice, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4) {333; CHECK-LABEL: za_write_vg4_vert_b:334; CHECK: // %bb.0:335; CHECK-NEXT: mov w12, w0336; CHECK-NEXT: mov za0v.b[w12, 0:3], { z0.b - z3.b }337; CHECK-NEXT: mov za0v.b[w12, 12:15], { z0.b - z3.b }338; CHECK-NEXT: ret339 call void @llvm.aarch64.sme.write.ver.vg4.nxv16i8(i32 0, i32 %slice, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4)340 %slice.12 = add i32 %slice, 12341 call void @llvm.aarch64.sme.write.ver.vg4.nxv16i8(i32 0, i32 %slice.12, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4)342 ret void343}344 345define void @za_write_vg4_vert_h(i32 %slice, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4) {346; CHECK-LABEL: za_write_vg4_vert_h:347; CHECK: // %bb.0:348; CHECK-NEXT: mov w12, w0349; CHECK-NEXT: mov za0v.h[w12, 0:3], { z0.h - z3.h }350; CHECK-NEXT: mov za1v.h[w12, 4:7], { z0.h - z3.h }351; CHECK-NEXT: ret352 call void @llvm.aarch64.sme.write.ver.vg4.nxv8i16(i32 0, i32 %slice, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4)353 %slice.4 = add i32 %slice, 4354 call void @llvm.aarch64.sme.write.ver.vg4.nxv8i16(i32 1, i32 %slice.4, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4)355 ret void356}357 358define void @za_write_vg4_vert_f16(i32 %slice, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3, <vscale x 8 x half> %zn4) {359; CHECK-LABEL: za_write_vg4_vert_f16:360; CHECK: // %bb.0:361; CHECK-NEXT: mov w12, w0362; CHECK-NEXT: mov za0v.h[w12, 0:3], { z0.h - z3.h }363; CHECK-NEXT: mov za1v.h[w12, 4:7], { z0.h - z3.h }364; CHECK-NEXT: ret365 call void @llvm.aarch64.sme.write.ver.vg4.nxv8f16(i32 0, i32 %slice, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3, <vscale x 8 x half> %zn4)366 %slice.4 = add i32 %slice, 4367 call void @llvm.aarch64.sme.write.ver.vg4.nxv8f16(i32 1, i32 %slice.4, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zn3, <vscale x 8 x half> %zn4)368 ret void369}370 371define void @za_write_vg4_vert_bf16(i32 %slice, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3, <vscale x 8 x bfloat> %zn4) {372; CHECK-LABEL: za_write_vg4_vert_bf16:373; CHECK: // %bb.0:374; CHECK-NEXT: mov w12, w0375; CHECK-NEXT: mov za0v.h[w12, 0:3], { z0.h - z3.h }376; CHECK-NEXT: mov za1v.h[w12, 4:7], { z0.h - z3.h }377; CHECK-NEXT: ret378 call void @llvm.aarch64.sme.write.ver.vg4.nxv8bf16(i32 0, i32 %slice, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3, <vscale x 8 x bfloat> %zn4)379 %slice.4 = add i32 %slice, 4380 call void @llvm.aarch64.sme.write.ver.vg4.nxv8bf16(i32 1, i32 %slice.4, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zn3, <vscale x 8 x bfloat> %zn4)381 ret void382}383 384define void @za_write_vg4_vert_s(i32 %slice, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4) {385; CHECK-LABEL: za_write_vg4_vert_s:386; CHECK: // %bb.0:387; CHECK-NEXT: mov w12, w0388; CHECK-NEXT: mov za0v.s[w12, 0:3], { z0.s - z3.s }389; CHECK-NEXT: ret390 call void @llvm.aarch64.sme.write.ver.vg4.nxv4i32(i32 0, i32 %slice, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4)391 ret void392}393 394define void @za_write_vg4_vert_f32(i32 %slice, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3, <vscale x 4 x float> %zn4) {395; CHECK-LABEL: za_write_vg4_vert_f32:396; CHECK: // %bb.0:397; CHECK-NEXT: mov w12, w0398; CHECK-NEXT: mov za0v.s[w12, 0:3], { z0.s - z3.s }399; CHECK-NEXT: ret400 call void @llvm.aarch64.sme.write.ver.vg4.nxv4f32(i32 0, i32 %slice, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3, <vscale x 4 x float> %zn4)401 ret void402}403 404define void @za_write_vg4_vert_d(i32 %slice, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4) {405; CHECK-LABEL: za_write_vg4_vert_d:406; CHECK: // %bb.0:407; CHECK-NEXT: mov w12, w0408; CHECK-NEXT: mov za0v.d[w12, 0:3], { z0.d - z3.d }409; CHECK-NEXT: ret410 call void @llvm.aarch64.sme.write.ver.vg4.nxv2i64(i32 0, i32 %slice, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3, <vscale x 2 x i64> %zn4)411 ret void412}413 414define void @za_write_vg4_vert_f64(i32 %slice, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3, <vscale x 2 x double> %zn4) {415; CHECK-LABEL: za_write_vg4_vert_f64:416; CHECK: // %bb.0:417; CHECK-NEXT: mov w12, w0418; CHECK-NEXT: mov za0v.d[w12, 0:3], { z0.d - z3.d }419; CHECK-NEXT: ret420 call void @llvm.aarch64.sme.write.ver.vg4.nxv2f64(i32 0, i32 %slice, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3, <vscale x 2 x double> %zn4)421 ret void422}423 424define void @za_write_vg4_vert_f64_tuple(i64 %stride, ptr %ptr) {425; CHECK-LABEL: za_write_vg4_vert_f64_tuple:426; CHECK: // %bb.0: // %entry427; CHECK-NEXT: lsl x8, x0, #1428; CHECK-NEXT: add x9, x1, x0429; CHECK-NEXT: ptrue pn8.b430; CHECK-NEXT: ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x1]431; CHECK-NEXT: ld1d { z17.d, z21.d, z25.d, z29.d }, pn8/z, [x9]432; CHECK-NEXT: mov w12, wzr433; CHECK-NEXT: add x10, x1, x8434; CHECK-NEXT: add x8, x9, x8435; CHECK-NEXT: ld1d { z18.d, z22.d, z26.d, z30.d }, pn8/z, [x10]436; CHECK-NEXT: ld1d { z19.d, z23.d, z27.d, z31.d }, pn8/z, [x8]437; CHECK-NEXT: mov za0v.d[w12, 0:3], { z16.d - z19.d }438; CHECK-NEXT: mov za0v.d[w12, 0:3], { z20.d - z23.d }439; CHECK-NEXT: mov za0v.d[w12, 0:3], { z24.d - z27.d }440; CHECK-NEXT: mov za0v.d[w12, 0:3], { z28.d - z31.d }441; CHECK-NEXT: ret442entry:443 %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()444 %1 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %0, ptr %ptr)445 %2 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %1, 0446 %3 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %1, 1447 %4 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %1, 2448 %5 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %1, 3449 %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride450 %6 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %0, ptr %arrayidx2)451 %7 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %6, 0452 %8 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %6, 1453 %9 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %6, 2454 %10 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %6, 3455 %mul3 = shl i64 %stride, 1456 %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3457 %11 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %0, ptr %arrayidx4)458 %12 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %11, 0459 %13 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %11, 1460 %14 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %11, 2461 %15 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %11, 3462 %mul5 = mul i64 %stride, 3463 %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5464 %16 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %0, ptr %arrayidx6)465 %17 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %16, 0466 %18 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %16, 1467 %19 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %16, 2468 %20 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %16, 3469 call void @llvm.aarch64.sme.write.ver.vg4.nxv2f64(i32 0, i32 0, <vscale x 2 x double> %2, <vscale x 2 x double> %7, <vscale x 2 x double> %12, <vscale x 2 x double> %17)470 call void @llvm.aarch64.sme.write.ver.vg4.nxv2f64(i32 0, i32 0, <vscale x 2 x double> %3, <vscale x 2 x double> %8, <vscale x 2 x double> %13, <vscale x 2 x double> %18)471 call void @llvm.aarch64.sme.write.ver.vg4.nxv2f64(i32 0, i32 0, <vscale x 2 x double> %4, <vscale x 2 x double> %9, <vscale x 2 x double> %14, <vscale x 2 x double> %19)472 call void @llvm.aarch64.sme.write.ver.vg4.nxv2f64(i32 0, i32 0, <vscale x 2 x double> %5, <vscale x 2 x double> %10, <vscale x 2 x double> %15, <vscale x 2 x double> %20)473 ret void474}475 476;477; Move Multi-Vector To ZA (Write) x2478;479 480define void @za_write_vg1x2_b(i32 %slice, <vscale x 16 x i8> %za1, <vscale x 16 x i8> %za2) {481; CHECK-LABEL: za_write_vg1x2_b:482; CHECK: // %bb.0:483; CHECK-NEXT: mov w8, w0484; CHECK-NEXT: mov za.d[w8, 0, vgx2], { z0.d, z1.d }485; CHECK-NEXT: mov za.d[w8, 7, vgx2], { z0.d, z1.d }486; CHECK-NEXT: ret487 call void @llvm.aarch64.sme.write.vg1x2.nxv16i8(i32 %slice, <vscale x 16 x i8> %za1, <vscale x 16 x i8> %za2)488 %slice.7 = add i32 %slice, 7489 call void @llvm.aarch64.sme.write.vg1x2.nxv16i8(i32 %slice.7, <vscale x 16 x i8> %za1, <vscale x 16 x i8> %za2)490 ret void491}492 493define void @za_write_vg1x2_b_tuple(i64 %stride, ptr %ptr) {494; CHECK-LABEL: za_write_vg1x2_b_tuple:495; CHECK: // %bb.0: // %entry496; CHECK-NEXT: ptrue pn8.b497; CHECK-NEXT: mov w8, wzr498; CHECK-NEXT: ld1b { z16.b, z24.b }, pn8/z, [x1]499; CHECK-NEXT: ld1b { z17.b, z25.b }, pn8/z, [x1, x0]500; CHECK-NEXT: mov za.d[w8, 0, vgx2], { z16.d, z17.d }501; CHECK-NEXT: mov za.d[w8, 0, vgx2], { z24.d, z25.d }502; CHECK-NEXT: ret503entry:504 %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()505 %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)506 %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 0507 %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 1508 %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride509 %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)510 %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 0511 %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 1512 call void @llvm.aarch64.sme.write.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5)513 call void @llvm.aarch64.sme.write.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6)514 ret void515}516 517define void @za_write_vg1x2_h(i32 %slice, <vscale x 8 x i16> %za1, <vscale x 8 x i16> %za2) {518; CHECK-LABEL: za_write_vg1x2_h:519; CHECK: // %bb.0:520; CHECK-NEXT: mov w8, w0521; CHECK-NEXT: mov za.d[w8, 0, vgx2], { z0.d, z1.d }522; CHECK-NEXT: mov za.d[w8, 7, vgx2], { z0.d, z1.d }523; CHECK-NEXT: ret524 call void @llvm.aarch64.sme.write.vg1x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %za1, <vscale x 8 x i16> %za2)525 %slice.7 = add i32 %slice, 7526 call void @llvm.aarch64.sme.write.vg1x2.nxv8i16(i32 %slice.7, <vscale x 8 x i16> %za1, <vscale x 8 x i16> %za2)527 ret void528}529 530define void @za_write_vg1x2_f16(i32 %slice, <vscale x 8 x half> %za1, <vscale x 8 x half> %za2) {531; CHECK-LABEL: za_write_vg1x2_f16:532; CHECK: // %bb.0:533; CHECK-NEXT: mov w8, w0534; CHECK-NEXT: mov za.d[w8, 0, vgx2], { z0.d, z1.d }535; CHECK-NEXT: mov za.d[w8, 7, vgx2], { z0.d, z1.d }536; CHECK-NEXT: ret537 call void @llvm.aarch64.sme.write.vg1x2.nxv8f16(i32 %slice, <vscale x 8 x half> %za1, <vscale x 8 x half> %za2)538 %slice.7 = add i32 %slice, 7539 call void @llvm.aarch64.sme.write.vg1x2.nxv8f16(i32 %slice.7, <vscale x 8 x half> %za1, <vscale x 8 x half> %za2)540 ret void541}542 543define void @za_write_vg1x2_bf16(i32 %slice, <vscale x 8 x bfloat> %za1, <vscale x 8 x bfloat> %za2) {544; CHECK-LABEL: za_write_vg1x2_bf16:545; CHECK: // %bb.0:546; CHECK-NEXT: mov w8, w0547; CHECK-NEXT: mov za.d[w8, 0, vgx2], { z0.d, z1.d }548; CHECK-NEXT: mov za.d[w8, 7, vgx2], { z0.d, z1.d }549; CHECK-NEXT: ret550 call void @llvm.aarch64.sme.write.vg1x2.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %za1, <vscale x 8 x bfloat> %za2)551 %slice.7 = add i32 %slice, 7552 call void @llvm.aarch64.sme.write.vg1x2.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %za1, <vscale x 8 x bfloat> %za2)553 ret void554}555 556define void @za_write_vg1x2_s(i32 %slice, <vscale x 4 x i32> %za1, <vscale x 4 x i32> %za2) {557; CHECK-LABEL: za_write_vg1x2_s:558; CHECK: // %bb.0:559; CHECK-NEXT: mov w8, w0560; CHECK-NEXT: mov za.d[w8, 0, vgx2], { z0.d, z1.d }561; CHECK-NEXT: mov za.d[w8, 7, vgx2], { z0.d, z1.d }562; CHECK-NEXT: ret563 call void @llvm.aarch64.sme.write.vg1x2.nxv4i32(i32 %slice, <vscale x 4 x i32> %za1, <vscale x 4 x i32> %za2)564 %slice.7 = add i32 %slice, 7565 call void @llvm.aarch64.sme.write.vg1x2.nxv4i32(i32 %slice.7, <vscale x 4 x i32> %za1, <vscale x 4 x i32> %za2)566 ret void567}568 569define void @za_write_vg1x2_f32(i32 %slice, <vscale x 4 x float> %za1, <vscale x 4 x float> %za2) {570; CHECK-LABEL: za_write_vg1x2_f32:571; CHECK: // %bb.0:572; CHECK-NEXT: mov w8, w0573; CHECK-NEXT: mov za.d[w8, 0, vgx2], { z0.d, z1.d }574; CHECK-NEXT: mov za.d[w8, 7, vgx2], { z0.d, z1.d }575; CHECK-NEXT: ret576 call void @llvm.aarch64.sme.write.vg1x2.nxv4f32(i32 %slice, <vscale x 4 x float> %za1, <vscale x 4 x float> %za2)577 %slice.7 = add i32 %slice, 7578 call void @llvm.aarch64.sme.write.vg1x2.nxv4f32(i32 %slice.7, <vscale x 4 x float> %za1, <vscale x 4 x float> %za2)579 ret void580}581 582define void @za_write_vg1x2_d(i32 %slice, <vscale x 2 x i64> %za1, <vscale x 2 x i64> %za2) {583; CHECK-LABEL: za_write_vg1x2_d:584; CHECK: // %bb.0:585; CHECK-NEXT: mov w8, w0586; CHECK-NEXT: mov za.d[w8, 0, vgx2], { z0.d, z1.d }587; CHECK-NEXT: mov za.d[w8, 7, vgx2], { z0.d, z1.d }588; CHECK-NEXT: ret589 call void @llvm.aarch64.sme.write.vg1x2.nxv2i64(i32 %slice, <vscale x 2 x i64> %za1, <vscale x 2 x i64> %za2)590 %slice.7 = add i32 %slice, 7591 call void @llvm.aarch64.sme.write.vg1x2.nxv2i64(i32 %slice.7, <vscale x 2 x i64> %za1, <vscale x 2 x i64> %za2)592 ret void593}594 595define void @za_write_vg1x2_f64(i32 %slice, <vscale x 2 x double> %za1, <vscale x 2 x double> %za2) {596; CHECK-LABEL: za_write_vg1x2_f64:597; CHECK: // %bb.0:598; CHECK-NEXT: mov w8, w0599; CHECK-NEXT: mov za.d[w8, 0, vgx2], { z0.d, z1.d }600; CHECK-NEXT: mov za.d[w8, 7, vgx2], { z0.d, z1.d }601; CHECK-NEXT: ret602 call void @llvm.aarch64.sme.write.vg1x2.nxv2f64(i32 %slice, <vscale x 2 x double> %za1, <vscale x 2 x double> %za2)603 %slice.7 = add i32 %slice, 7604 call void @llvm.aarch64.sme.write.vg1x2.nxv2f64(i32 %slice.7, <vscale x 2 x double> %za1, <vscale x 2 x double> %za2)605 ret void606}607 608;609; Move Multi-Vector To ZA (Write) x4610;611 612define void @za_write_vg1x4_b(i32 %slice, <vscale x 16 x i8> %za1, <vscale x 16 x i8> %za2, <vscale x 16 x i8> %za3, <vscale x 16 x i8> %za4) {613; CHECK-LABEL: za_write_vg1x4_b:614; CHECK: // %bb.0:615; CHECK-NEXT: mov w8, w0616; CHECK-NEXT: mov za.d[w8, 0, vgx4], { z0.d - z3.d }617; CHECK-NEXT: mov za.d[w8, 7, vgx4], { z0.d - z3.d }618; CHECK-NEXT: ret619 call void @llvm.aarch64.sme.write.vg1x4.nxv16i8(i32 %slice, <vscale x 16 x i8> %za1, <vscale x 16 x i8> %za2, <vscale x 16 x i8> %za3, <vscale x 16 x i8> %za4)620 %slice.7 = add i32 %slice, 7621 call void @llvm.aarch64.sme.write.vg1x4.nxv16i8(i32 %slice.7, <vscale x 16 x i8> %za1, <vscale x 16 x i8> %za2, <vscale x 16 x i8> %za3, <vscale x 16 x i8> %za4)622 ret void623}624 625define void @za_write_vg1x4_h(i32 %slice, <vscale x 8 x i16> %za1, <vscale x 8 x i16> %za2, <vscale x 8 x i16> %za3, <vscale x 8 x i16> %za4) {626; CHECK-LABEL: za_write_vg1x4_h:627; CHECK: // %bb.0:628; CHECK-NEXT: mov w8, w0629; CHECK-NEXT: mov za.d[w8, 0, vgx4], { z0.d - z3.d }630; CHECK-NEXT: mov za.d[w8, 7, vgx4], { z0.d - z3.d }631; CHECK-NEXT: ret632 call void @llvm.aarch64.sme.write.vg1x4.nxv8i16(i32 %slice, <vscale x 8 x i16> %za1, <vscale x 8 x i16> %za2, <vscale x 8 x i16> %za3, <vscale x 8 x i16> %za4)633 %slice.7 = add i32 %slice, 7634 call void @llvm.aarch64.sme.write.vg1x4.nxv8i16(i32 %slice.7, <vscale x 8 x i16> %za1, <vscale x 8 x i16> %za2, <vscale x 8 x i16> %za3, <vscale x 8 x i16> %za4)635 ret void636}637 638define void @za_write_vg1x4_f16(i32 %slice, <vscale x 8 x half> %za1, <vscale x 8 x half> %za2, <vscale x 8 x half> %za3, <vscale x 8 x half> %za4) {639; CHECK-LABEL: za_write_vg1x4_f16:640; CHECK: // %bb.0:641; CHECK-NEXT: mov w8, w0642; CHECK-NEXT: mov za.d[w8, 0, vgx4], { z0.d - z3.d }643; CHECK-NEXT: mov za.d[w8, 7, vgx4], { z0.d - z3.d }644; CHECK-NEXT: ret645 call void @llvm.aarch64.sme.write.vg1x4.nxv8f16(i32 %slice, <vscale x 8 x half> %za1, <vscale x 8 x half> %za2, <vscale x 8 x half> %za3, <vscale x 8 x half> %za4)646 %slice.7 = add i32 %slice, 7647 call void @llvm.aarch64.sme.write.vg1x4.nxv8f16(i32 %slice.7, <vscale x 8 x half> %za1, <vscale x 8 x half> %za2, <vscale x 8 x half> %za3, <vscale x 8 x half> %za4)648 ret void649}650 651define void @za_write_vg1x4_bf16(i32 %slice, <vscale x 8 x bfloat> %za1, <vscale x 8 x bfloat> %za2, <vscale x 8 x bfloat> %za3, <vscale x 8 x bfloat> %za4) {652; CHECK-LABEL: za_write_vg1x4_bf16:653; CHECK: // %bb.0:654; CHECK-NEXT: mov w8, w0655; CHECK-NEXT: mov za.d[w8, 0, vgx4], { z0.d - z3.d }656; CHECK-NEXT: mov za.d[w8, 7, vgx4], { z0.d - z3.d }657; CHECK-NEXT: ret658 call void @llvm.aarch64.sme.write.vg1x4.nxv8bf16(i32 %slice, <vscale x 8 x bfloat> %za1, <vscale x 8 x bfloat> %za2, <vscale x 8 x bfloat> %za3, <vscale x 8 x bfloat> %za4)659 %slice.7 = add i32 %slice, 7660 call void @llvm.aarch64.sme.write.vg1x4.nxv8bf16(i32 %slice.7, <vscale x 8 x bfloat> %za1, <vscale x 8 x bfloat> %za2, <vscale x 8 x bfloat> %za3, <vscale x 8 x bfloat> %za4)661 ret void662}663 664define void @za_write_vg1x4_s(i32 %slice, <vscale x 4 x i32> %za1, <vscale x 4 x i32> %za2, <vscale x 4 x i32> %za3, <vscale x 4 x i32> %za4) {665; CHECK-LABEL: za_write_vg1x4_s:666; CHECK: // %bb.0:667; CHECK-NEXT: mov w8, w0668; CHECK-NEXT: mov za.d[w8, 0, vgx4], { z0.d - z3.d }669; CHECK-NEXT: mov za.d[w8, 7, vgx4], { z0.d - z3.d }670; CHECK-NEXT: ret671 call void @llvm.aarch64.sme.write.vg1x4.nxv4i32(i32 %slice, <vscale x 4 x i32> %za1, <vscale x 4 x i32> %za2, <vscale x 4 x i32> %za3, <vscale x 4 x i32> %za4)672 %slice.7 = add i32 %slice, 7673 call void @llvm.aarch64.sme.write.vg1x4.nxv4i32(i32 %slice.7, <vscale x 4 x i32> %za1, <vscale x 4 x i32> %za2, <vscale x 4 x i32> %za3, <vscale x 4 x i32> %za4)674 ret void675}676 677define void @za_write_vg1x4_f32(i32 %slice, <vscale x 4 x float> %za1, <vscale x 4 x float> %za2, <vscale x 4 x float> %za3, <vscale x 4 x float> %za4) {678; CHECK-LABEL: za_write_vg1x4_f32:679; CHECK: // %bb.0:680; CHECK-NEXT: mov w8, w0681; CHECK-NEXT: mov za.d[w8, 0, vgx4], { z0.d - z3.d }682; CHECK-NEXT: mov za.d[w8, 7, vgx4], { z0.d - z3.d }683; CHECK-NEXT: ret684 call void @llvm.aarch64.sme.write.vg1x4.nxv4f32(i32 %slice, <vscale x 4 x float> %za1, <vscale x 4 x float> %za2, <vscale x 4 x float> %za3, <vscale x 4 x float> %za4)685 %slice.7 = add i32 %slice, 7686 call void @llvm.aarch64.sme.write.vg1x4.nxv4f32(i32 %slice.7, <vscale x 4 x float> %za1, <vscale x 4 x float> %za2, <vscale x 4 x float> %za3, <vscale x 4 x float> %za4)687 ret void688}689 690define void @za_write_vg1x4_d(i32 %slice, <vscale x 2 x i64> %za1, <vscale x 2 x i64> %za2, <vscale x 2 x i64> %za3, <vscale x 2 x i64> %za4) {691; CHECK-LABEL: za_write_vg1x4_d:692; CHECK: // %bb.0:693; CHECK-NEXT: mov w8, w0694; CHECK-NEXT: mov za.d[w8, 0, vgx4], { z0.d - z3.d }695; CHECK-NEXT: mov za.d[w8, 7, vgx4], { z0.d - z3.d }696; CHECK-NEXT: ret697 call void @llvm.aarch64.sme.write.vg1x4.nxv2i64(i32 %slice, <vscale x 2 x i64> %za1, <vscale x 2 x i64> %za2, <vscale x 2 x i64> %za3, <vscale x 2 x i64> %za4)698 %slice.7 = add i32 %slice, 7699 call void @llvm.aarch64.sme.write.vg1x4.nxv2i64(i32 %slice.7, <vscale x 2 x i64> %za1, <vscale x 2 x i64> %za2, <vscale x 2 x i64> %za3, <vscale x 2 x i64> %za4)700 ret void701}702 703define void @za_write_vg1x4_f64(i32 %slice, <vscale x 2 x double> %za1, <vscale x 2 x double> %za2, <vscale x 2 x double> %za3, <vscale x 2 x double> %za4) {704; CHECK-LABEL: za_write_vg1x4_f64:705; CHECK: // %bb.0:706; CHECK-NEXT: mov w8, w0707; CHECK-NEXT: mov za.d[w8, 0, vgx4], { z0.d - z3.d }708; CHECK-NEXT: mov za.d[w8, 7, vgx4], { z0.d - z3.d }709; CHECK-NEXT: ret710 call void @llvm.aarch64.sme.write.vg1x4.nxv2f64(i32 %slice, <vscale x 2 x double> %za1, <vscale x 2 x double> %za2, <vscale x 2 x double> %za3, <vscale x 2 x double> %za4)711 %slice.7 = add i32 %slice, 7712 call void @llvm.aarch64.sme.write.vg1x4.nxv2f64(i32 %slice.7, <vscale x 2 x double> %za1, <vscale x 2 x double> %za2, <vscale x 2 x double> %za3, <vscale x 2 x double> %za4)713 ret void714}715 716define void @za_write_vg1x4_f64_tuple(i64 %stride, ptr %ptr) {717; CHECK-LABEL: za_write_vg1x4_f64_tuple:718; CHECK: // %bb.0: // %entry719; CHECK-NEXT: lsl x9, x0, #1720; CHECK-NEXT: add x10, x1, x0721; CHECK-NEXT: ptrue pn8.b722; CHECK-NEXT: ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x1]723; CHECK-NEXT: ld1d { z17.d, z21.d, z25.d, z29.d }, pn8/z, [x10]724; CHECK-NEXT: mov w8, wzr725; CHECK-NEXT: add x11, x1, x9726; CHECK-NEXT: add x9, x10, x9727; CHECK-NEXT: ld1d { z18.d, z22.d, z26.d, z30.d }, pn8/z, [x11]728; CHECK-NEXT: ld1d { z19.d, z23.d, z27.d, z31.d }, pn8/z, [x9]729; CHECK-NEXT: mov za.d[w8, 0, vgx4], { z16.d - z19.d }730; CHECK-NEXT: mov za.d[w8, 0, vgx4], { z20.d - z23.d }731; CHECK-NEXT: mov za.d[w8, 0, vgx4], { z24.d - z27.d }732; CHECK-NEXT: mov za.d[w8, 0, vgx4], { z28.d - z31.d }733; CHECK-NEXT: ret734entry:735 %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()736 %1 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %0, ptr %ptr)737 %2 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %1, 0738 %3 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %1, 1739 %4 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %1, 2740 %5 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %1, 3741 %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride742 %6 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %0, ptr %arrayidx2)743 %7 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %6, 0744 %8 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %6, 1745 %9 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %6, 2746 %10 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %6, 3747 %mul3 = shl i64 %stride, 1748 %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3749 %11 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %0, ptr %arrayidx4)750 %12 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %11, 0751 %13 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %11, 1752 %14 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %11, 2753 %15 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %11, 3754 %mul5 = mul i64 %stride, 3755 %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5756 %16 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %0, ptr %arrayidx6)757 %17 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %16, 0758 %18 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %16, 1759 %19 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %16, 2760 %20 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %16, 3761 call void @llvm.aarch64.sme.write.vg1x4.nxv2f64(i32 0, <vscale x 2 x double> %2, <vscale x 2 x double> %7, <vscale x 2 x double> %12, <vscale x 2 x double> %17)762 call void @llvm.aarch64.sme.write.vg1x4.nxv2f64(i32 0, <vscale x 2 x double> %3, <vscale x 2 x double> %8, <vscale x 2 x double> %13, <vscale x 2 x double> %18)763 call void @llvm.aarch64.sme.write.vg1x4.nxv2f64(i32 0, <vscale x 2 x double> %4, <vscale x 2 x double> %9, <vscale x 2 x double> %14, <vscale x 2 x double> %19)764 call void @llvm.aarch64.sme.write.vg1x4.nxv2f64(i32 0, <vscale x 2 x double> %5, <vscale x 2 x double> %10, <vscale x 2 x double> %15, <vscale x 2 x double> %20)765 ret void766}767 768declare void @llvm.aarch64.sme.write.hor.vg2.nxv16i8(i32, i32, <vscale x 16 x i8>, <vscale x 16 x i8>)769declare void @llvm.aarch64.sme.write.hor.vg2.nxv8i16(i32, i32, <vscale x 8 x i16>, <vscale x 8 x i16>)770declare void @llvm.aarch64.sme.write.hor.vg2.nxv8f16(i32, i32, <vscale x 8 x half>, <vscale x 8 x half>)771declare void @llvm.aarch64.sme.write.hor.vg2.nxv8bf16(i32, i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)772declare void @llvm.aarch64.sme.write.hor.vg2.nxv4i32(i32, i32, <vscale x 4 x i32>, <vscale x 4 x i32>)773declare void @llvm.aarch64.sme.write.hor.vg2.nxv4f32(i32, i32, <vscale x 4 x float>, <vscale x 4 x float>)774declare void @llvm.aarch64.sme.write.hor.vg2.nxv2i64(i32, i32, <vscale x 2 x i64>, <vscale x 2 x i64>)775declare void @llvm.aarch64.sme.write.hor.vg2.nxv2f64(i32, i32, <vscale x 2 x double>, <vscale x 2 x double>)776 777declare void @llvm.aarch64.sme.write.ver.vg2.nxv16i8(i32, i32, <vscale x 16 x i8>, <vscale x 16 x i8>)778declare void @llvm.aarch64.sme.write.ver.vg2.nxv8i16(i32, i32, <vscale x 8 x i16>, <vscale x 8 x i16>)779declare void @llvm.aarch64.sme.write.ver.vg2.nxv8f16(i32, i32, <vscale x 8 x half>, <vscale x 8 x half>)780declare void @llvm.aarch64.sme.write.ver.vg2.nxv8bf16(i32, i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)781declare void @llvm.aarch64.sme.write.ver.vg2.nxv4i32(i32, i32, <vscale x 4 x i32>, <vscale x 4 x i32>)782declare void @llvm.aarch64.sme.write.ver.vg2.nxv4f32(i32, i32, <vscale x 4 x float>, <vscale x 4 x float>)783declare void @llvm.aarch64.sme.write.ver.vg2.nxv2i64(i32, i32, <vscale x 2 x i64>, <vscale x 2 x i64>)784declare void @llvm.aarch64.sme.write.ver.vg2.nxv2f64(i32, i32, <vscale x 2 x double>, <vscale x 2 x double>)785 786declare void @llvm.aarch64.sme.write.hor.vg4.nxv16i8(i32, i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)787declare void @llvm.aarch64.sme.write.hor.vg4.nxv8i16(i32, i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)788declare void @llvm.aarch64.sme.write.hor.vg4.nxv8f16(i32, i32, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)789declare void @llvm.aarch64.sme.write.hor.vg4.nxv8bf16(i32, i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)790declare void @llvm.aarch64.sme.write.hor.vg4.nxv4i32(i32, i32, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)791declare void @llvm.aarch64.sme.write.hor.vg4.nxv4f32(i32, i32, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)792declare void @llvm.aarch64.sme.write.hor.vg4.nxv2i64(i32, i32, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)793declare void @llvm.aarch64.sme.write.hor.vg4.nxv2f64(i32, i32, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)794 795declare void @llvm.aarch64.sme.write.ver.vg4.nxv16i8(i32, i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)796declare void @llvm.aarch64.sme.write.ver.vg4.nxv8i16(i32, i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)797declare void @llvm.aarch64.sme.write.ver.vg4.nxv8f16(i32, i32, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)798declare void @llvm.aarch64.sme.write.ver.vg4.nxv8bf16(i32, i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)799declare void @llvm.aarch64.sme.write.ver.vg4.nxv4i32(i32, i32, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)800declare void @llvm.aarch64.sme.write.ver.vg4.nxv4f32(i32, i32, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)801declare void @llvm.aarch64.sme.write.ver.vg4.nxv2i64(i32, i32, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)802declare void @llvm.aarch64.sme.write.ver.vg4.nxv2f64(i32, i32, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)803 804declare void @llvm.aarch64.sme.write.vg1x2.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>)805declare void @llvm.aarch64.sme.write.vg1x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>)806declare void @llvm.aarch64.sme.write.vg1x2.nxv4i32(i32, <vscale x 4 x i32>, <vscale x 4 x i32>)807declare void @llvm.aarch64.sme.write.vg1x2.nxv2i64(i32, <vscale x 2 x i64>, <vscale x 2 x i64>)808declare void @llvm.aarch64.sme.write.vg1x2.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>)809declare void @llvm.aarch64.sme.write.vg1x2.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)810declare void @llvm.aarch64.sme.write.vg1x2.nxv4f32(i32, <vscale x 4 x float>, <vscale x 4 x float>)811declare void @llvm.aarch64.sme.write.vg1x2.nxv2f64(i32, <vscale x 2 x double>, <vscale x 2 x double>)812 813declare void @llvm.aarch64.sme.write.vg1x4.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)814declare void @llvm.aarch64.sme.write.vg1x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)815declare void @llvm.aarch64.sme.write.vg1x4.nxv4i32(i32, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)816declare void @llvm.aarch64.sme.write.vg1x4.nxv2i64(i32, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)817declare void @llvm.aarch64.sme.write.vg1x4.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)818declare void @llvm.aarch64.sme.write.vg1x4.nxv8bf16(i32, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)819declare void @llvm.aarch64.sme.write.vg1x4.nxv4f32(i32, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)820declare void @llvm.aarch64.sme.write.vg1x4.nxv2f64(i32, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)821