809 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2,+sme-b16b16 -verify-machineinstrs -force-streaming < %s | FileCheck %s3 4;5; Move Multi-Vector From Tile (Read) x26;7 8; Horizontal9 10define <vscale x 16 x i8> @za_read_horiz_vg2_b(i32 %slice) {11; CHECK-LABEL: za_read_horiz_vg2_b:12; CHECK: // %bb.0:13; CHECK-NEXT: mov w12, w014; CHECK-NEXT: mov { z0.b, z1.b }, za0h.b[w12, 0:1]15; CHECK-NEXT: mov { z2.b, z3.b }, za0h.b[w12, 14:15]16; CHECK-NEXT: add z0.b, z0.b, z2.b17; CHECK-NEXT: ret18 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.hor.vg2.nxv16i8(i32 0, i32 %slice)19 %slice.14 = add i32 %slice, 1420 %res2 = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.hor.vg2.nxv16i8(i32 0, i32 %slice.14)21 %val1 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %res, 022 %val2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %res2, 023 %sum = add <vscale x 16 x i8> %val1, %val224 ret <vscale x 16 x i8> %sum25}26 27define <vscale x 8 x i16> @za_read_horiz_vg2_h(i32 %slice) {28; CHECK-LABEL: za_read_horiz_vg2_h:29; CHECK: // %bb.0:30; CHECK-NEXT: mov w12, w031; CHECK-NEXT: mov { z0.h, z1.h }, za0h.h[w12, 0:1]32; CHECK-NEXT: mov { z2.h, z3.h }, za1h.h[w12, 6:7]33; CHECK-NEXT: add z0.h, z0.h, z2.h34; CHECK-NEXT: ret35 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.hor.vg2.nxv8i16(i32 0, i32 %slice)36 %slice.6 = add i32 %slice, 637 %res2 = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.hor.vg2.nxv8i16(i32 1, i32 %slice.6)38 %val1 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %res, 039 %val2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %res2, 040 %sum = add <vscale x 8 x i16> %val1, %val241 ret <vscale x 8 x i16> %sum42}43 44define <vscale x 8 x half> @za_read_horiz_vg2_f16(i32 %slice) {45; CHECK-LABEL: za_read_horiz_vg2_f16:46; CHECK: // %bb.0:47; CHECK-NEXT: mov w12, w048; CHECK-NEXT: mov { z0.h, z1.h }, za0h.h[w12, 0:1]49; CHECK-NEXT: mov { z2.h, z3.h }, za1h.h[w12, 6:7]50; CHECK-NEXT: fadd z0.h, z0.h, z2.h51; CHECK-NEXT: ret52 %res = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.hor.vg2.nxv8f16(i32 0, i32 %slice)53 %slice.6 = add i32 %slice, 654 %res2 = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.hor.vg2.nxv8f16(i32 1, i32 %slice.6)55 %val1 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %res, 056 %val2 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %res2, 057 %sum = fadd <vscale x 8 x half> %val1, %val258 ret <vscale x 8 x half> %sum59}60 61define <vscale x 8 x bfloat> @za_read_horiz_vg2_bf16(i32 %slice) {62; CHECK-LABEL: za_read_horiz_vg2_bf16:63; CHECK: // %bb.0:64; CHECK-NEXT: mov w12, w065; CHECK-NEXT: mov { z0.h, z1.h }, za0h.h[w12, 0:1]66; CHECK-NEXT: mov { z2.h, z3.h }, za1h.h[w12, 6:7]67; CHECK-NEXT: bfadd z0.h, z0.h, z2.h68; CHECK-NEXT: ret69 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.hor.vg2.nxv8bf16(i32 0, i32 %slice)70 %slice.6 = add i32 %slice, 671 %res2 = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.hor.vg2.nxv8bf16(i32 1, i32 %slice.6)72 %val1 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res, 073 %val2 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res2, 074 %sum = fadd <vscale x 8 x bfloat> %val1, %val275 ret <vscale x 8 x bfloat> %sum76}77 78define <vscale x 4 x i32> @za_read_horiz_vg2_s(i32 %slice) {79; CHECK-LABEL: za_read_horiz_vg2_s:80; CHECK: // %bb.0:81; CHECK-NEXT: mov w12, w082; CHECK-NEXT: mov { z0.s, z1.s }, za0h.s[w12, 0:1]83; CHECK-NEXT: mov { z2.s, z3.s }, za3h.s[w12, 2:3]84; CHECK-NEXT: add z0.s, z0.s, z2.s85; CHECK-NEXT: ret86 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sme.read.hor.vg2.nxv4i32(i32 0, i32 %slice)87 %slice.2 = add i32 %slice, 288 %res2 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sme.read.hor.vg2.nxv4i32(i32 3, i32 %slice.2)89 %val1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %res, 090 %val2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %res2, 091 %sum = add <vscale x 4 x i32> %val1, %val292 ret <vscale x 4 x i32> %sum93}94 95define <vscale x 4 x float> @za_read_horiz_vg2_f32(i32 %slice) {96; CHECK-LABEL: za_read_horiz_vg2_f32:97; CHECK: // %bb.0:98; CHECK-NEXT: mov w12, w099; CHECK-NEXT: mov { z0.s, z1.s }, za0h.s[w12, 0:1]100; CHECK-NEXT: mov { z2.s, z3.s }, za3h.s[w12, 2:3]101; CHECK-NEXT: fadd z0.s, z0.s, z2.s102; CHECK-NEXT: ret103 %res = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sme.read.hor.vg2.nxv4f32(i32 0, i32 %slice)104 %slice.2 = add i32 %slice, 2105 %res2 = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sme.read.hor.vg2.nxv4f32(i32 3, i32 %slice.2)106 %val1 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %res, 0107 %val2 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %res2, 0108 %sum = fadd <vscale x 4 x float> %val1, %val2109 ret <vscale x 4 x float> %sum110}111 112define { <vscale x 2 x i64>, <vscale x 2 x i64> } @za_read_horiz_vg2_d(i32 %slice) {113; CHECK-LABEL: za_read_horiz_vg2_d:114; CHECK: // %bb.0:115; CHECK-NEXT: mov w12, w0116; CHECK-NEXT: mov { z0.d, z1.d }, za0h.d[w12, 0:1]117; CHECK-NEXT: ret118 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sme.read.hor.vg2.nxv2i64(i32 0, i32 %slice)119 ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res120}121 122define { <vscale x 2 x double>, <vscale x 2 x double> } @za_read_horiz_vg2_f64(i32 %slice) {123; CHECK-LABEL: za_read_horiz_vg2_f64:124; CHECK: // %bb.0:125; CHECK-NEXT: mov w12, w0126; CHECK-NEXT: mov { z0.d, z1.d }, za0h.d[w12, 0:1]127; CHECK-NEXT: ret128 %res = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sme.read.hor.vg2.nxv2f64(i32 0, i32 %slice)129 ret { <vscale x 2 x double>, <vscale x 2 x double> } %res130}131 132; Vertical133 134define <vscale x 16 x i8> @za_read_vert_vg2_b(i32 %slice) {135; CHECK-LABEL: za_read_vert_vg2_b:136; CHECK: // %bb.0:137; CHECK-NEXT: mov w12, w0138; CHECK-NEXT: mov { z0.b, z1.b }, za0v.b[w12, 0:1]139; CHECK-NEXT: mov { z2.b, z3.b }, za0v.b[w12, 14:15]140; CHECK-NEXT: add z0.b, z0.b, z2.b141; CHECK-NEXT: ret142 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.ver.vg2.nxv16i8(i32 0, i32 %slice)143 %slice.14 = add i32 %slice, 14144 %res2 = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.ver.vg2.nxv16i8(i32 0, i32 %slice.14)145 %val1 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %res, 0146 %val2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %res2, 0147 %sum = add <vscale x 16 x i8> %val1, %val2148 ret <vscale x 16 x i8> %sum149}150 151define <vscale x 8 x i16> @za_read_vert_vg2_h(i32 %slice) {152; CHECK-LABEL: za_read_vert_vg2_h:153; CHECK: // %bb.0:154; CHECK-NEXT: mov w12, w0155; CHECK-NEXT: mov { z0.h, z1.h }, za0v.h[w12, 0:1]156; CHECK-NEXT: mov { z2.h, z3.h }, za1v.h[w12, 6:7]157; CHECK-NEXT: add z0.h, z0.h, z2.h158; CHECK-NEXT: ret159 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.ver.vg2.nxv8i16(i32 0, i32 %slice)160 %slice.6 = add i32 %slice, 6161 %res2 = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.ver.vg2.nxv8i16(i32 1, i32 %slice.6)162 %val1 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %res, 0163 %val2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %res2, 0164 %sum = add <vscale x 8 x i16> %val1, %val2165 ret <vscale x 8 x i16> %sum166}167 168define <vscale x 8 x half> @za_read_vert_vg2_f16(i32 %slice) {169; CHECK-LABEL: za_read_vert_vg2_f16:170; CHECK: // %bb.0:171; CHECK-NEXT: mov w12, w0172; CHECK-NEXT: mov { z0.h, z1.h }, za0v.h[w12, 0:1]173; CHECK-NEXT: mov { z2.h, z3.h }, za1v.h[w12, 6:7]174; CHECK-NEXT: fadd z0.h, z0.h, z2.h175; CHECK-NEXT: ret176 %res = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.ver.vg2.nxv8f16(i32 0, i32 %slice)177 %slice.6 = add i32 %slice, 6178 %res2 = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.ver.vg2.nxv8f16(i32 1, i32 %slice.6)179 %val1 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %res, 0180 %val2 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %res2, 0181 %sum = fadd <vscale x 8 x half> %val1, %val2182 ret <vscale x 8 x half> %sum183}184 185define <vscale x 8 x bfloat> @za_read_vert_vg2_bf16(i32 %slice) {186; CHECK-LABEL: za_read_vert_vg2_bf16:187; CHECK: // %bb.0:188; CHECK-NEXT: mov w12, w0189; CHECK-NEXT: mov { z0.h, z1.h }, za0v.h[w12, 0:1]190; CHECK-NEXT: mov { z2.h, z3.h }, za1v.h[w12, 6:7]191; CHECK-NEXT: bfadd z0.h, z0.h, z2.h192; CHECK-NEXT: ret193 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.ver.vg2.nxv8bf16(i32 0, i32 %slice)194 %slice.6 = add i32 %slice, 6195 %res2 = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.ver.vg2.nxv8bf16(i32 1, i32 %slice.6)196 %val1 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res, 0197 %val2 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res2, 0198 %sum = fadd <vscale x 8 x bfloat> %val1, %val2199 ret <vscale x 8 x bfloat> %sum200}201 202define <vscale x 4 x i32> @za_read_vert_vg2_s(i32 %slice) {203; CHECK-LABEL: za_read_vert_vg2_s:204; CHECK: // %bb.0:205; CHECK-NEXT: mov w12, w0206; CHECK-NEXT: mov { z0.s, z1.s }, za0v.s[w12, 0:1]207; CHECK-NEXT: mov { z2.s, z3.s }, za3v.s[w12, 2:3]208; CHECK-NEXT: add z0.s, z0.s, z2.s209; CHECK-NEXT: ret210 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sme.read.ver.vg2.nxv4i32(i32 0, i32 %slice)211 %slice.2 = add i32 %slice, 2212 %res2 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sme.read.ver.vg2.nxv4i32(i32 3, i32 %slice.2)213 %val1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %res, 0214 %val2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %res2, 0215 %sum = add <vscale x 4 x i32> %val1, %val2216 ret <vscale x 4 x i32> %sum217}218 219define <vscale x 4 x float> @za_read_vert_vg2_f32(i32 %slice) {220; CHECK-LABEL: za_read_vert_vg2_f32:221; CHECK: // %bb.0:222; CHECK-NEXT: mov w12, w0223; CHECK-NEXT: mov { z0.s, z1.s }, za0v.s[w12, 0:1]224; CHECK-NEXT: mov { z2.s, z3.s }, za3v.s[w12, 2:3]225; CHECK-NEXT: fadd z0.s, z0.s, z2.s226; CHECK-NEXT: ret227 %res = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sme.read.ver.vg2.nxv4f32(i32 0, i32 %slice)228 %slice.2 = add i32 %slice, 2229 %res2 = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sme.read.ver.vg2.nxv4f32(i32 3, i32 %slice.2)230 %val1 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %res, 0231 %val2 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %res2, 0232 %sum = fadd <vscale x 4 x float> %val1, %val2233 ret <vscale x 4 x float> %sum234}235 236define { <vscale x 2 x i64>, <vscale x 2 x i64> } @za_read_vert_vg2_d(i32 %slice) {237; CHECK-LABEL: za_read_vert_vg2_d:238; CHECK: // %bb.0:239; CHECK-NEXT: mov w12, w0240; CHECK-NEXT: mov { z0.d, z1.d }, za0v.d[w12, 0:1]241; CHECK-NEXT: ret242 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sme.read.ver.vg2.nxv2i64(i32 0, i32 %slice)243 ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res244}245 246define { <vscale x 2 x double>, <vscale x 2 x double> } @za_read_vert_vg2_f64(i32 %slice) {247; CHECK-LABEL: za_read_vert_vg2_f64:248; CHECK: // %bb.0:249; CHECK-NEXT: mov w12, w0250; CHECK-NEXT: mov { z0.d, z1.d }, za0v.d[w12, 0:1]251; CHECK-NEXT: ret252 %res = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sme.read.ver.vg2.nxv2f64(i32 0, i32 %slice)253 ret { <vscale x 2 x double>, <vscale x 2 x double> } %res254}255 256;257; Move Multi-Vector From Tile (Read) x4258;259 260; Horizontal261 262define <vscale x 16 x i8> @za_read_horiz_vg4_b(i32 %slice) {263; CHECK-LABEL: za_read_horiz_vg4_b:264; CHECK: // %bb.0:265; CHECK-NEXT: mov w12, w0266; CHECK-NEXT: mov { z0.b - z3.b }, za0h.b[w12, 0:3]267; CHECK-NEXT: mov { z4.b - z7.b }, za0h.b[w12, 12:15]268; CHECK-NEXT: add z0.b, z0.b, z4.b269; CHECK-NEXT: ret270 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.hor.vg4.nxv16i8(i32 0, i32 %slice)271 %slice.12 = add i32 %slice, 12272 %res2 = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.hor.vg4.nxv16i8(i32 0, i32 %slice.12)273 %val1 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res, 0274 %val2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res2, 0275 %sum = add <vscale x 16 x i8> %val1, %val2276 ret <vscale x 16 x i8> %sum277}278 279define <vscale x 8 x i16> @za_read_horiz_vg4_h(i32 %slice) {280; CHECK-LABEL: za_read_horiz_vg4_h:281; CHECK: // %bb.0:282; CHECK-NEXT: mov w12, w0283; CHECK-NEXT: mov { z0.h - z3.h }, za0h.h[w12, 0:3]284; CHECK-NEXT: mov { z4.h - z7.h }, za1h.h[w12, 4:7]285; CHECK-NEXT: add z0.h, z0.h, z4.h286; CHECK-NEXT: ret287 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.hor.vg4.nxv8i16(i32 0, i32 %slice)288 %slice.4 = add i32 %slice, 4289 %res2 = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.hor.vg4.nxv8i16(i32 1, i32 %slice.4)290 %val1 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res, 0291 %val2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res2, 0292 %sum = add <vscale x 8 x i16> %val1, %val2293 ret <vscale x 8 x i16> %sum294}295 296define <vscale x 8 x half> @za_read_horiz_vg4_f16(i32 %slice) {297; CHECK-LABEL: za_read_horiz_vg4_f16:298; CHECK: // %bb.0:299; CHECK-NEXT: mov w12, w0300; CHECK-NEXT: mov { z0.h - z3.h }, za0h.h[w12, 0:3]301; CHECK-NEXT: mov { z4.h - z7.h }, za1h.h[w12, 4:7]302; CHECK-NEXT: fadd z0.h, z0.h, z4.h303; CHECK-NEXT: ret304 %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.hor.vg4.nxv8f16(i32 0, i32 %slice)305 %slice.4 = add i32 %slice, 4306 %res2 = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.hor.vg4.nxv8f16(i32 1, i32 %slice.4)307 %val1 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res, 0308 %val2 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res2, 0309 %sum = fadd <vscale x 8 x half> %val1, %val2310 ret <vscale x 8 x half> %sum311}312 313define <vscale x 8 x bfloat> @za_read_horiz_vg4_bf16(i32 %slice) {314; CHECK-LABEL: za_read_horiz_vg4_bf16:315; CHECK: // %bb.0:316; CHECK-NEXT: mov w12, w0317; CHECK-NEXT: mov { z0.h - z3.h }, za0h.h[w12, 0:3]318; CHECK-NEXT: mov { z4.h - z7.h }, za1h.h[w12, 4:7]319; CHECK-NEXT: bfadd z0.h, z0.h, z4.h320; CHECK-NEXT: ret321 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.hor.vg4.nxv8bf16(i32 0, i32 %slice)322 %slice.4 = add i32 %slice, 4323 %res2 = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.hor.vg4.nxv8bf16(i32 1, i32 %slice.4)324 %val1 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res, 0325 %val2 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res2, 0326 %sum = fadd <vscale x 8 x bfloat> %val1, %val2327 ret <vscale x 8 x bfloat> %sum328}329 330define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @za_read_horiz_vg4_s(i32 %slice) {331; CHECK-LABEL: za_read_horiz_vg4_s:332; CHECK: // %bb.0:333; CHECK-NEXT: mov w12, w0334; CHECK-NEXT: mov { z0.s - z3.s }, za0h.s[w12, 0:3]335; CHECK-NEXT: ret336 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sme.read.hor.vg4.nxv4i32(i32 0, i32 %slice)337 ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res338}339 340define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @za_read_horiz_vg4_f32(i32 %slice) {341; CHECK-LABEL: za_read_horiz_vg4_f32:342; CHECK: // %bb.0:343; CHECK-NEXT: mov w12, w0344; CHECK-NEXT: mov { z0.s - z3.s }, za0h.s[w12, 0:3]345; CHECK-NEXT: ret346 %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sme.read.hor.vg4.nxv4f32(i32 0, i32 %slice)347 ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res348}349 350define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @za_read_horiz_vg4_d(i32 %slice) {351; CHECK-LABEL: za_read_horiz_vg4_d:352; CHECK: // %bb.0:353; CHECK-NEXT: mov w12, w0354; CHECK-NEXT: mov { z0.d - z3.d }, za0h.d[w12, 0:3]355; CHECK-NEXT: ret356 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sme.read.hor.vg4.nxv2i64(i32 0, i32 %slice)357 ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res358}359 360define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @za_read_horiz_vg4_f64(i32 %slice) {361; CHECK-LABEL: za_read_horiz_vg4_f64:362; CHECK: // %bb.0:363; CHECK-NEXT: mov w12, w0364; CHECK-NEXT: mov { z0.d - z3.d }, za0h.d[w12, 0:3]365; CHECK-NEXT: ret366 %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sme.read.hor.vg4.nxv2f64(i32 0, i32 %slice)367 ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res368}369 370; Vertical371 372define <vscale x 16 x i8> @za_read_vert_vg4_b(i32 %slice) {373; CHECK-LABEL: za_read_vert_vg4_b:374; CHECK: // %bb.0:375; CHECK-NEXT: mov w12, w0376; CHECK-NEXT: mov { z0.b - z3.b }, za0v.b[w12, 0:3]377; CHECK-NEXT: mov { z4.b - z7.b }, za0v.b[w12, 12:15]378; CHECK-NEXT: add z0.b, z0.b, z4.b379; CHECK-NEXT: ret380 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.ver.vg4.nxv16i8(i32 0, i32 %slice)381 %slice.12 = add i32 %slice, 12382 %res2 = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.ver.vg4.nxv16i8(i32 0, i32 %slice.12)383 %val1 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res, 0384 %val2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res2, 0385 %sum = add <vscale x 16 x i8> %val1, %val2386 ret <vscale x 16 x i8> %sum387}388 389define <vscale x 8 x i16> @za_read_vert_vg4_h(i32 %slice) {390; CHECK-LABEL: za_read_vert_vg4_h:391; CHECK: // %bb.0:392; CHECK-NEXT: mov w12, w0393; CHECK-NEXT: mov { z0.h - z3.h }, za0v.h[w12, 0:3]394; CHECK-NEXT: mov { z4.h - z7.h }, za1v.h[w12, 4:7]395; CHECK-NEXT: add z0.h, z0.h, z4.h396; CHECK-NEXT: ret397 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.ver.vg4.nxv8i16(i32 0, i32 %slice)398 %slice.4 = add i32 %slice, 4399 %res2 = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.ver.vg4.nxv8i16(i32 1, i32 %slice.4)400 %val1 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res, 0401 %val2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res2, 0402 %sum = add <vscale x 8 x i16> %val1, %val2403 ret <vscale x 8 x i16> %sum404}405 406define <vscale x 8 x half> @za_read_vert_vg4_f16(i32 %slice) {407; CHECK-LABEL: za_read_vert_vg4_f16:408; CHECK: // %bb.0:409; CHECK-NEXT: mov w12, w0410; CHECK-NEXT: mov { z0.h - z3.h }, za0v.h[w12, 0:3]411; CHECK-NEXT: mov { z4.h - z7.h }, za1v.h[w12, 4:7]412; CHECK-NEXT: fadd z0.h, z0.h, z4.h413; CHECK-NEXT: ret414 %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.ver.vg4.nxv8f16(i32 0, i32 %slice)415 %slice.4 = add i32 %slice, 4416 %res2 = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.ver.vg4.nxv8f16(i32 1, i32 %slice.4)417 %val1 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res, 0418 %val2 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res2, 0419 %sum = fadd <vscale x 8 x half> %val1, %val2420 ret <vscale x 8 x half> %sum421}422 423define <vscale x 8 x bfloat> @za_read_vert_vg4_bf16(i32 %slice) {424; CHECK-LABEL: za_read_vert_vg4_bf16:425; CHECK: // %bb.0:426; CHECK-NEXT: mov w12, w0427; CHECK-NEXT: mov { z0.h - z3.h }, za0v.h[w12, 0:3]428; CHECK-NEXT: mov { z4.h - z7.h }, za1v.h[w12, 4:7]429; CHECK-NEXT: bfadd z0.h, z0.h, z4.h430; CHECK-NEXT: ret431 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.ver.vg4.nxv8bf16(i32 0, i32 %slice)432 %slice.4 = add i32 %slice, 4433 %res2 = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.ver.vg4.nxv8bf16(i32 1, i32 %slice.4)434 %val1 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res, 0435 %val2 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res2, 0436 %sum = fadd <vscale x 8 x bfloat> %val1, %val2437 ret <vscale x 8 x bfloat> %sum438}439 440define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @za_read_vert_vg4_s(i32 %slice) {441; CHECK-LABEL: za_read_vert_vg4_s:442; CHECK: // %bb.0:443; CHECK-NEXT: mov w12, w0444; CHECK-NEXT: mov { z0.s - z3.s }, za0v.s[w12, 0:3]445; CHECK-NEXT: ret446 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sme.read.ver.vg4.nxv4i32(i32 0, i32 %slice)447 ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res448}449 450define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @za_read_vert_vg4_f32(i32 %slice) {451; CHECK-LABEL: za_read_vert_vg4_f32:452; CHECK: // %bb.0:453; CHECK-NEXT: mov w12, w0454; CHECK-NEXT: mov { z0.s - z3.s }, za0v.s[w12, 0:3]455; CHECK-NEXT: ret456 %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sme.read.ver.vg4.nxv4f32(i32 0, i32 %slice)457 ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res458}459 460define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @za_read_vert_vg4_d(i32 %slice) {461; CHECK-LABEL: za_read_vert_vg4_d:462; CHECK: // %bb.0:463; CHECK-NEXT: mov w12, w0464; CHECK-NEXT: mov { z0.d - z3.d }, za0v.d[w12, 0:3]465; CHECK-NEXT: ret466 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sme.read.ver.vg4.nxv2i64(i32 0, i32 %slice)467 ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res468}469 470define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @za_read_vert_vg4_f64(i32 %slice) {471; CHECK-LABEL: za_read_vert_vg4_f64:472; CHECK: // %bb.0:473; CHECK-NEXT: mov w12, w0474; CHECK-NEXT: mov { z0.d - z3.d }, za0v.d[w12, 0:3]475; CHECK-NEXT: ret476 %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sme.read.ver.vg4.nxv2f64(i32 0, i32 %slice)477 ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res478}479 480; Move Multi-Vector From ZA (Read) x2481 482define <vscale x 16 x i8> @za_read_vg1x2_b(i32 %slice) {483; CHECK-LABEL: za_read_vg1x2_b:484; CHECK: // %bb.0:485; CHECK-NEXT: mov w8, w0486; CHECK-NEXT: mov { z0.d, z1.d }, za.d[w8, 0, vgx2]487; CHECK-NEXT: mov { z2.d, z3.d }, za.d[w8, 7, vgx2]488; CHECK-NEXT: add z0.b, z0.b, z2.b489; CHECK-NEXT: ret490 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.vg1x2.nxv16i8(i32 %slice)491 %slice.7 = add i32 %slice, 7492 %res2 = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.vg1x2.nxv16i8(i32 %slice.7)493 %val1 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %res, 0494 %val2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %res2, 0495 %sum = add <vscale x 16 x i8> %val1, %val2496 ret <vscale x 16 x i8> %sum497}498 499define <vscale x 8 x i16> @za_read_vg1x2_h(i32 %slice) {500; CHECK-LABEL: za_read_vg1x2_h:501; CHECK: // %bb.0:502; CHECK-NEXT: mov w8, w0503; CHECK-NEXT: mov { z0.d, z1.d }, za.d[w8, 0, vgx2]504; CHECK-NEXT: mov { z2.d, z3.d }, za.d[w8, 7, vgx2]505; CHECK-NEXT: add z0.h, z0.h, z2.h506; CHECK-NEXT: ret507 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.vg1x2.nxv8i16(i32 %slice)508 %slice.7 = add i32 %slice, 7509 %res2 = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.vg1x2.nxv8i16(i32 %slice.7)510 %val1 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %res, 0511 %val2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %res2, 0512 %sum = add <vscale x 8 x i16> %val1, %val2513 ret <vscale x 8 x i16> %sum514}515 516define <vscale x 8 x half> @za_read_vg1x2_f16(i32 %slice) {517; CHECK-LABEL: za_read_vg1x2_f16:518; CHECK: // %bb.0:519; CHECK-NEXT: mov w8, w0520; CHECK-NEXT: mov { z0.d, z1.d }, za.d[w8, 0, vgx2]521; CHECK-NEXT: mov { z2.d, z3.d }, za.d[w8, 7, vgx2]522; CHECK-NEXT: fadd z0.h, z0.h, z2.h523; CHECK-NEXT: ret524 %res = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.vg1x2.nxv8f16(i32 %slice)525 %slice.7 = add i32 %slice, 7526 %res2 = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.vg1x2.nxv8f16(i32 %slice.7)527 %val1 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %res, 0528 %val2 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } %res2, 0529 %sum = fadd <vscale x 8 x half> %val1, %val2530 ret <vscale x 8 x half> %sum531}532 533define <vscale x 8 x bfloat> @za_read_vg1x2_bf16(i32 %slice) {534; CHECK-LABEL: za_read_vg1x2_bf16:535; CHECK: // %bb.0:536; CHECK-NEXT: mov w8, w0537; CHECK-NEXT: mov { z0.d, z1.d }, za.d[w8, 0, vgx2]538; CHECK-NEXT: mov { z2.d, z3.d }, za.d[w8, 7, vgx2]539; CHECK-NEXT: bfadd z0.h, z0.h, z2.h540; CHECK-NEXT: ret541 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.vg1x2.nxv8bf16(i32 %slice)542 %slice.7 = add i32 %slice, 7543 %res2 = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.vg1x2.nxv8bf16(i32 %slice.7)544 %val1 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res, 0545 %val2 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res2, 0546 %sum = fadd <vscale x 8 x bfloat> %val1, %val2547 ret <vscale x 8 x bfloat> %sum548}549 550define <vscale x 4 x i32> @za_read_vg1x2_s(i32 %slice) {551; CHECK-LABEL: za_read_vg1x2_s:552; CHECK: // %bb.0:553; CHECK-NEXT: mov w8, w0554; CHECK-NEXT: mov { z0.d, z1.d }, za.d[w8, 0, vgx2]555; CHECK-NEXT: mov { z2.d, z3.d }, za.d[w8, 7, vgx2]556; CHECK-NEXT: add z0.s, z0.s, z2.s557; CHECK-NEXT: ret558 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sme.read.vg1x2.nxv4i32(i32 %slice)559 %slice.7 = add i32 %slice, 7560 %res2 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sme.read.vg1x2.nxv4i32(i32 %slice.7)561 %val1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %res, 0562 %val2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %res2, 0563 %sum = add <vscale x 4 x i32> %val1, %val2564 ret <vscale x 4 x i32> %sum565}566 567define <vscale x 4 x float> @za_read_vg1x2_f32(i32 %slice) {568; CHECK-LABEL: za_read_vg1x2_f32:569; CHECK: // %bb.0:570; CHECK-NEXT: mov w8, w0571; CHECK-NEXT: mov { z0.d, z1.d }, za.d[w8, 0, vgx2]572; CHECK-NEXT: mov { z2.d, z3.d }, za.d[w8, 7, vgx2]573; CHECK-NEXT: fadd z0.s, z0.s, z2.s574; CHECK-NEXT: ret575 %res = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sme.read.vg1x2.nxv4f32(i32 %slice)576 %slice.7 = add i32 %slice, 7577 %res2 = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sme.read.vg1x2.nxv4f32(i32 %slice.7)578 %val1 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %res, 0579 %val2 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %res2, 0580 %sum = fadd <vscale x 4 x float> %val1, %val2581 ret <vscale x 4 x float> %sum582}583 584define <vscale x 2 x i64> @za_read_vg1x2_d(i32 %slice) {585; CHECK-LABEL: za_read_vg1x2_d:586; CHECK: // %bb.0:587; CHECK-NEXT: mov w8, w0588; CHECK-NEXT: mov { z0.d, z1.d }, za.d[w8, 0, vgx2]589; CHECK-NEXT: mov { z2.d, z3.d }, za.d[w8, 7, vgx2]590; CHECK-NEXT: add z0.d, z0.d, z2.d591; CHECK-NEXT: ret592 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sme.read.vg1x2.nxv2i64(i32 %slice)593 %slice.7 = add i32 %slice, 7594 %res2 = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sme.read.vg1x2.nxv2i64(i32 %slice.7)595 %val1 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } %res, 0596 %val2 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } %res2, 0597 %sum = add <vscale x 2 x i64> %val1, %val2598 ret <vscale x 2 x i64> %sum599}600 601define <vscale x 2 x double> @za_read_vg1x2_f64(i32 %slice) {602; CHECK-LABEL: za_read_vg1x2_f64:603; CHECK: // %bb.0:604; CHECK-NEXT: mov w8, w0605; CHECK-NEXT: mov { z0.d, z1.d }, za.d[w8, 0, vgx2]606; CHECK-NEXT: mov { z2.d, z3.d }, za.d[w8, 7, vgx2]607; CHECK-NEXT: fadd z0.d, z0.d, z2.d608; CHECK-NEXT: ret609 %res = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sme.read.vg1x2.nxv2f64(i32 %slice)610 %slice.7 = add i32 %slice, 7611 %res2 = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sme.read.vg1x2.nxv2f64(i32 %slice.7)612 %val1 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %res, 0613 %val2 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %res2, 0614 %sum = fadd <vscale x 2 x double> %val1, %val2615 ret <vscale x 2 x double> %sum616}617 618; Move Multi-Vector From ZA (Read) x4619 620define <vscale x 16 x i8> @za_read_vg1x4_b(i32 %slice) {621; CHECK-LABEL: za_read_vg1x4_b:622; CHECK: // %bb.0:623; CHECK-NEXT: mov w8, w0624; CHECK-NEXT: mov { z0.d - z3.d }, za.d[w8, 0, vgx4]625; CHECK-NEXT: mov { z4.d - z7.d }, za.d[w8, 7, vgx4]626; CHECK-NEXT: add z0.b, z0.b, z4.b627; CHECK-NEXT: ret628 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.vg1x4.nxv16i8(i32 %slice)629 %slice.7 = add i32 %slice, 7630 %res2 = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.vg1x4.nxv16i8(i32 %slice.7)631 %val1 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res, 0632 %val2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res2, 0633 %sum = add <vscale x 16 x i8> %val1, %val2634 ret <vscale x 16 x i8> %sum635}636 637define <vscale x 8 x i16> @za_read_vg1x4_h(i32 %slice) {638; CHECK-LABEL: za_read_vg1x4_h:639; CHECK: // %bb.0:640; CHECK-NEXT: mov w8, w0641; CHECK-NEXT: mov { z0.d - z3.d }, za.d[w8, 0, vgx4]642; CHECK-NEXT: mov { z4.d - z7.d }, za.d[w8, 7, vgx4]643; CHECK-NEXT: add z0.h, z0.h, z4.h644; CHECK-NEXT: ret645 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.vg1x4.nxv8i16(i32 %slice)646 %slice.7 = add i32 %slice, 7647 %res2 = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.vg1x4.nxv8i16(i32 %slice.7)648 %val1 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res, 0649 %val2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res2, 0650 %sum = add <vscale x 8 x i16> %val1, %val2651 ret <vscale x 8 x i16> %sum652}653 654define <vscale x 8 x half> @za_read_vg1x4_f16(i32 %slice) {655; CHECK-LABEL: za_read_vg1x4_f16:656; CHECK: // %bb.0:657; CHECK-NEXT: mov w8, w0658; CHECK-NEXT: mov { z0.d - z3.d }, za.d[w8, 0, vgx4]659; CHECK-NEXT: mov { z4.d - z7.d }, za.d[w8, 7, vgx4]660; CHECK-NEXT: fadd z0.h, z0.h, z4.h661; CHECK-NEXT: ret662 %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.vg1x4.nxv8f16(i32 %slice)663 %slice.7 = add i32 %slice, 7664 %res2 = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.vg1x4.nxv8f16(i32 %slice.7)665 %val1 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res, 0666 %val2 = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res2, 0667 %sum = fadd <vscale x 8 x half> %val1, %val2668 ret <vscale x 8 x half> %sum669}670 671define <vscale x 8 x bfloat> @za_read_vg1x4_bf16(i32 %slice) {672; CHECK-LABEL: za_read_vg1x4_bf16:673; CHECK: // %bb.0:674; CHECK-NEXT: mov w8, w0675; CHECK-NEXT: mov { z0.d - z3.d }, za.d[w8, 0, vgx4]676; CHECK-NEXT: mov { z4.d - z7.d }, za.d[w8, 7, vgx4]677; CHECK-NEXT: bfadd z0.h, z0.h, z4.h678; CHECK-NEXT: ret679 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.vg1x4.nxv8bf16(i32 %slice)680 %slice.7 = add i32 %slice, 7681 %res2 = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.vg1x4.nxv8bf16(i32 %slice.7)682 %val1 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res, 0683 %val2 = extractvalue { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res2, 0684 %sum = fadd <vscale x 8 x bfloat> %val1, %val2685 ret <vscale x 8 x bfloat> %sum686}687 688define <vscale x 4 x i32> @za_read_vg1x4_s(i32 %slice) {689; CHECK-LABEL: za_read_vg1x4_s:690; CHECK: // %bb.0:691; CHECK-NEXT: mov w8, w0692; CHECK-NEXT: mov { z0.d - z3.d }, za.d[w8, 0, vgx4]693; CHECK-NEXT: mov { z4.d - z7.d }, za.d[w8, 7, vgx4]694; CHECK-NEXT: add z0.s, z0.s, z4.s695; CHECK-NEXT: ret696 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sme.read.vg1x4.nxv4i32(i32 %slice)697 %slice.7 = add i32 %slice, 7698 %res2 = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sme.read.vg1x4.nxv4i32(i32 %slice.7)699 %val1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res, 0700 %val2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res2, 0701 %sum = add <vscale x 4 x i32> %val1, %val2702 ret <vscale x 4 x i32> %sum703}704 705define <vscale x 4 x float> @za_read_vg1x4_f32(i32 %slice) {706; CHECK-LABEL: za_read_vg1x4_f32:707; CHECK: // %bb.0:708; CHECK-NEXT: mov w8, w0709; CHECK-NEXT: mov { z0.d - z3.d }, za.d[w8, 0, vgx4]710; CHECK-NEXT: mov { z4.d - z7.d }, za.d[w8, 7, vgx4]711; CHECK-NEXT: fadd z0.s, z0.s, z4.s712; CHECK-NEXT: ret713 %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sme.read.vg1x4.nxv4f32(i32 %slice)714 %slice.7 = add i32 %slice, 7715 %res2 = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sme.read.vg1x4.nxv4f32(i32 %slice.7)716 %val1 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res, 0717 %val2 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res2, 0718 %sum = fadd <vscale x 4 x float> %val1, %val2719 ret <vscale x 4 x float> %sum720}721 722define <vscale x 2 x i64> @za_read_vg1x4_d(i32 %slice) {723; CHECK-LABEL: za_read_vg1x4_d:724; CHECK: // %bb.0:725; CHECK-NEXT: mov w8, w0726; CHECK-NEXT: mov { z0.d - z3.d }, za.d[w8, 0, vgx4]727; CHECK-NEXT: mov { z4.d - z7.d }, za.d[w8, 7, vgx4]728; CHECK-NEXT: add z0.d, z0.d, z4.d729; CHECK-NEXT: ret730 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sme.read.vg1x4.nxv2i64(i32 %slice)731 %slice.7 = add i32 %slice, 7732 %res2 = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sme.read.vg1x4.nxv2i64(i32 %slice.7)733 %val1 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res, 0734 %val2 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res2, 0735 %sum = add <vscale x 2 x i64> %val1, %val2736 ret <vscale x 2 x i64> %sum737}738 739define <vscale x 2 x double> @za_read_vg1x4_f64(i32 %slice) {740; CHECK-LABEL: za_read_vg1x4_f64:741; CHECK: // %bb.0:742; CHECK-NEXT: mov w8, w0743; CHECK-NEXT: mov { z0.d - z3.d }, za.d[w8, 0, vgx4]744; CHECK-NEXT: mov { z4.d - z7.d }, za.d[w8, 7, vgx4]745; CHECK-NEXT: fadd z0.d, z0.d, z4.d746; CHECK-NEXT: ret747 %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sme.read.vg1x4.nxv2f64(i32 %slice)748 %slice.7 = add i32 %slice, 7749 %res2 = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sme.read.vg1x4.nxv2f64(i32 %slice.7)750 %val1 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res, 0751 %val2 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res2, 0752 %sum = fadd <vscale x 2 x double> %val1, %val2753 ret <vscale x 2 x double> %sum754}755 756declare { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.hor.vg2.nxv16i8(i32, i32)757declare { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.hor.vg2.nxv8i16(i32, i32)758declare { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.hor.vg2.nxv8f16(i32, i32)759declare { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.hor.vg2.nxv8bf16(i32, i32)760declare { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sme.read.hor.vg2.nxv4i32(i32, i32)761declare { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sme.read.hor.vg2.nxv4f32(i32, i32)762declare { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sme.read.hor.vg2.nxv2i64(i32, i32)763declare { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sme.read.hor.vg2.nxv2f64(i32, i32)764 765declare { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.hor.vg4.nxv16i8(i32, i32)766declare { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.hor.vg4.nxv8i16(i32, i32)767declare { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.hor.vg4.nxv8f16(i32, i32)768declare { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.hor.vg4.nxv8bf16(i32, i32)769declare { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sme.read.hor.vg4.nxv4i32(i32, i32)770declare { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sme.read.hor.vg4.nxv4f32(i32, i32)771declare { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sme.read.hor.vg4.nxv2i64(i32, i32)772declare { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sme.read.hor.vg4.nxv2f64(i32, i32)773 774declare { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.ver.vg2.nxv16i8(i32, i32)775declare { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.ver.vg2.nxv8i16(i32, i32)776declare { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.ver.vg2.nxv8f16(i32, i32)777declare { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.ver.vg2.nxv8bf16(i32, i32)778declare { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sme.read.ver.vg2.nxv4i32(i32, i32)779declare { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sme.read.ver.vg2.nxv4f32(i32, i32)780declare { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sme.read.ver.vg2.nxv2i64(i32, i32)781declare { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sme.read.ver.vg2.nxv2f64(i32, i32)782 783declare { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.ver.vg4.nxv16i8(i32, i32)784declare { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.ver.vg4.nxv8i16(i32, i32)785declare { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.ver.vg4.nxv8f16(i32, i32)786declare { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.ver.vg4.nxv8bf16(i32, i32)787declare { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sme.read.ver.vg4.nxv4i32(i32, i32)788declare { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sme.read.ver.vg4.nxv4f32(i32, i32)789declare { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sme.read.ver.vg4.nxv2i64(i32, i32)790declare { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sme.read.ver.vg4.nxv2f64(i32, i32)791 792declare { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.vg1x2.nxv16i8(i32)793declare { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.vg1x2.nxv8i16(i32)794declare { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sme.read.vg1x2.nxv4i32(i32)795declare { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sme.read.vg1x2.nxv2i64(i32)796declare { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.vg1x2.nxv8f16(i32)797declare { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.vg1x2.nxv8bf16(i32)798declare { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sme.read.vg1x2.nxv4f32(i32)799declare { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sme.read.vg1x2.nxv2f64(i32)800 801declare { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sme.read.vg1x4.nxv16i8(i32)802declare { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sme.read.vg1x4.nxv8i16(i32)803declare { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sme.read.vg1x4.nxv4i32(i32)804declare { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sme.read.vg1x4.nxv2i64(i32)805declare { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sme.read.vg1x4.nxv8f16(i32)806declare { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sme.read.vg1x4.nxv8bf16(i32)807declare { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sme.read.vg1x4.nxv4f32(i32)808declare { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sme.read.vg1x4.nxv2f64(i32)809