1125 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 42; RUN: llc -verify-machineinstrs -force-streaming < %s | FileCheck %s3 4target triple = "aarch64-linux"5 6;MOVAZ (tile to vector, Multi)7 8 9;;10; X2 - Horiz11;;12 13define {<vscale x 16 x i8>, <vscale x 16 x i8>} @test_readz_hor_z8_i8_x2(i32 %tile, i32 %slice) #0 {14; CHECK-LABEL: test_readz_hor_z8_i8_x2:15; CHECK: // %bb.0:16; CHECK-NEXT: mov w12, w117; CHECK-NEXT: movaz { z0.b, z1.b }, za0h.b[w12, 0:1]18; CHECK-NEXT: movaz { z0.b, z1.b }, za0h.b[w12, 14:15]19; CHECK-NEXT: ret20 %res = call {<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sme.readz.horiz.x2.nxv16i8(i32 0, i32 %slice)21 %slice.max = add i32 %slice, 1422 %res2 = call {<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sme.readz.horiz.x2.nxv16i8(i32 0, i32 %slice.max)23 ret {<vscale x 16 x i8>, <vscale x 16 x i8>} %res224}25define {<vscale x 8 x i16>, <vscale x 8 x i16>} @test_readz_hor_z16_i16_x2(i32 %slice) #0 {26; CHECK-LABEL: test_readz_hor_z16_i16_x2:27; CHECK: // %bb.0:28; CHECK-NEXT: mov w12, w029; CHECK-NEXT: movaz { z0.h, z1.h }, za0h.h[w12, 0:1]30; CHECK-NEXT: movaz { z0.h, z1.h }, za1h.h[w12, 6:7]31; CHECK-NEXT: ret32 %res = call {<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.aarch64.sme.readz.horiz.x2.nxv8i16(i32 0, i32 %slice)33 %slice.max = add i32 %slice, 634 %res2 = call {<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.aarch64.sme.readz.horiz.x2.nxv8i16(i32 1, i32 %slice.max)35 ret {<vscale x 8 x i16>, <vscale x 8 x i16>} %res236}37 38define {<vscale x 4 x i32>, <vscale x 4 x i32>} @test_readz_hor_z32_i32_x2(i32 %slice) #0 {39; CHECK-LABEL: test_readz_hor_z32_i32_x2:40; CHECK: // %bb.0:41; CHECK-NEXT: mov w12, w042; CHECK-NEXT: movaz { z0.s, z1.s }, za0h.s[w12, 0:1]43; CHECK-NEXT: movaz { z0.s, z1.s }, za3h.s[w12, 2:3]44; CHECK-NEXT: ret45 %res = call {<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.aarch64.sme.readz.horiz.x2.nxv4i32(i32 0, i32 %slice)46 %slice.max = add i32 %slice, 247 %res2 = call {<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.aarch64.sme.readz.horiz.x2.nxv4i32(i32 3, i32 %slice.max)48 ret {<vscale x 4 x i32>, <vscale x 4 x i32>} %res249}50 51define {<vscale x 2 x i64>, <vscale x 2 x i64>} @test_readz_hor_z64_i64_x2(i32 %slice) #0 {52; CHECK-LABEL: test_readz_hor_z64_i64_x2:53; CHECK: // %bb.0:54; CHECK-NEXT: mov w12, w055; CHECK-NEXT: movaz { z0.d, z1.d }, za0h.d[w12, 0:1]56; CHECK-NEXT: movaz { z2.d, z3.d }, za7h.d[w12, 0:1]57; CHECK-NEXT: ret58 %res = call {<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.aarch64.sme.readz.horiz.x2.nxv2i64(i32 0, i32 %slice)59 %res2 = call {<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.aarch64.sme.readz.horiz.x2.nxv2i64(i32 7, i32 %slice)60 ret {<vscale x 2 x i64>, <vscale x 2 x i64>} %res61}62 63define {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @test_readz_hor_z16_bf16_x2(i32 %slice) #0 {64; CHECK-LABEL: test_readz_hor_z16_bf16_x2:65; CHECK: // %bb.0:66; CHECK-NEXT: mov w12, w067; CHECK-NEXT: movaz { z0.h, z1.h }, za0h.h[w12, 0:1]68; CHECK-NEXT: movaz { z0.h, z1.h }, za1h.h[w12, 6:7]69; CHECK-NEXT: ret70 %res = call {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.aarch64.sme.readz.horiz.x2.nxv8bf16(i32 0, i32 %slice)71 %slice.max = add i32 %slice, 672 %res2 = call {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.aarch64.sme.readz.horiz.x2.nxv8bf16(i32 1, i32 %slice.max)73 ret {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} %res274}75 76define {<vscale x 8 x half>, <vscale x 8 x half>} @test_readz_hor_z16_f16_x2(i32 %slice) #0 {77; CHECK-LABEL: test_readz_hor_z16_f16_x2:78; CHECK: // %bb.0:79; CHECK-NEXT: mov w12, w080; CHECK-NEXT: movaz { z0.h, z1.h }, za0h.h[w12, 0:1]81; CHECK-NEXT: movaz { z0.h, z1.h }, za1h.h[w12, 6:7]82; CHECK-NEXT: ret83 %res = call {<vscale x 8 x half>, <vscale x 8 x half>} @llvm.aarch64.sme.readz.horiz.x2.nxv8f16(i32 0, i32 %slice)84 %slice.max = add i32 %slice, 685 %res2 = call {<vscale x 8 x half>, <vscale x 8 x half>} @llvm.aarch64.sme.readz.horiz.x2.nxv8f16(i32 1, i32 %slice.max)86 ret {<vscale x 8 x half>, <vscale x 8 x half>} %res287}88 89define {<vscale x 4 x float>, <vscale x 4 x float>} @test_readz_hor_z32_f32_x2(i32 %slice) #0 {90; CHECK-LABEL: test_readz_hor_z32_f32_x2:91; CHECK: // %bb.0:92; CHECK-NEXT: mov w12, w093; CHECK-NEXT: movaz { z0.s, z1.s }, za0h.s[w12, 0:1]94; CHECK-NEXT: movaz { z0.s, z1.s }, za3h.s[w12, 2:3]95; CHECK-NEXT: ret96 %res = call {<vscale x 4 x float>, <vscale x 4 x float>} @llvm.aarch64.sme.readz.horiz.x2.nxv4f32(i32 0, i32 %slice)97 %slice.max = add i32 %slice, 298 %res2 = call {<vscale x 4 x float>, <vscale x 4 x float>} @llvm.aarch64.sme.readz.horiz.x2.nxv4f32(i32 3, i32 %slice.max)99 ret {<vscale x 4 x float>, <vscale x 4 x float>} %res2100}101 102define {<vscale x 2 x double>, <vscale x 2 x double>} @test_readz_hor_z64_f64_x2(i32 %slice) #0 {103; CHECK-LABEL: test_readz_hor_z64_f64_x2:104; CHECK: // %bb.0:105; CHECK-NEXT: mov w12, w0106; CHECK-NEXT: movaz { z0.d, z1.d }, za0h.d[w12, 0:1]107; CHECK-NEXT: movaz { z2.d, z3.d }, za7h.d[w12, 0:1]108; CHECK-NEXT: ret109 %res = call {<vscale x 2 x double>, <vscale x 2 x double>} @llvm.aarch64.sme.readz.horiz.x2.nxv2f64(i32 0, i32 %slice)110 %res2 = call {<vscale x 2 x double>, <vscale x 2 x double>} @llvm.aarch64.sme.readz.horiz.x2.nxv2f64(i32 7, i32 %slice)111 ret {<vscale x 2 x double>, <vscale x 2 x double>} %res112}113 114;;115; X2- Vert116;;117 118define {<vscale x 16 x i8>, <vscale x 16 x i8>} @test_readz_ver_z8_i8_x2(i32 %tile, i32 %slice) #0 {119; CHECK-LABEL: test_readz_ver_z8_i8_x2:120; CHECK: // %bb.0:121; CHECK-NEXT: mov w12, w1122; CHECK-NEXT: movaz { z0.b, z1.b }, za0v.b[w12, 0:1]123; CHECK-NEXT: movaz { z0.b, z1.b }, za0v.b[w12, 14:15]124; CHECK-NEXT: ret125 %res = call {<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sme.readz.vert.x2.nxv16i8(i32 0, i32 %slice)126 %slice.max = add i32 %slice, 14127 %res2 = call {<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sme.readz.vert.x2.nxv16i8(i32 0, i32 %slice.max)128 ret {<vscale x 16 x i8>, <vscale x 16 x i8>} %res2129}130define {<vscale x 8 x i16>, <vscale x 8 x i16>} @test_readz_ver_z16_i16_x2(i32 %slice) #0 {131; CHECK-LABEL: test_readz_ver_z16_i16_x2:132; CHECK: // %bb.0:133; CHECK-NEXT: mov w12, w0134; CHECK-NEXT: movaz { z0.h, z1.h }, za0v.h[w12, 0:1]135; CHECK-NEXT: movaz { z0.h, z1.h }, za1v.h[w12, 6:7]136; CHECK-NEXT: ret137 %res = call {<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.aarch64.sme.readz.vert.x2.nxv8i16(i32 0, i32 %slice)138 %slice.max = add i32 %slice, 6139 %res2 = call {<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.aarch64.sme.readz.vert.x2.nxv8i16(i32 1, i32 %slice.max)140 ret {<vscale x 8 x i16>, <vscale x 8 x i16>} %res2141}142 143define {<vscale x 4 x i32>, <vscale x 4 x i32>} @test_readz_ver_z32_i32_x2(i32 %slice) #0 {144; CHECK-LABEL: test_readz_ver_z32_i32_x2:145; CHECK: // %bb.0:146; CHECK-NEXT: mov w12, w0147; CHECK-NEXT: movaz { z0.s, z1.s }, za0v.s[w12, 0:1]148; CHECK-NEXT: movaz { z0.s, z1.s }, za3v.s[w12, 2:3]149; CHECK-NEXT: ret150 %res = call {<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.aarch64.sme.readz.vert.x2.nxv4i32(i32 0, i32 %slice)151 %slice.max = add i32 %slice, 2152 %res2 = call {<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.aarch64.sme.readz.vert.x2.nxv4i32(i32 3, i32 %slice.max)153 ret {<vscale x 4 x i32>, <vscale x 4 x i32>} %res2154}155 156define {<vscale x 2 x i64>, <vscale x 2 x i64>} @test_readz_ver_z64_i64_x2(i32 %slice) #0 {157; CHECK-LABEL: test_readz_ver_z64_i64_x2:158; CHECK: // %bb.0:159; CHECK-NEXT: mov w12, w0160; CHECK-NEXT: movaz { z0.d, z1.d }, za0v.d[w12, 0:1]161; CHECK-NEXT: movaz { z2.d, z3.d }, za7v.d[w12, 0:1]162; CHECK-NEXT: ret163 %res = call {<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.aarch64.sme.readz.vert.x2.nxv2i64(i32 0, i32 %slice)164 %res2 = call {<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.aarch64.sme.readz.vert.x2.nxv2i64(i32 7, i32 %slice)165 ret {<vscale x 2 x i64>, <vscale x 2 x i64>} %res166}167 168define {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @test_readz_ver_z16_bf16_x2(i32 %slice) #0 {169; CHECK-LABEL: test_readz_ver_z16_bf16_x2:170; CHECK: // %bb.0:171; CHECK-NEXT: mov w12, w0172; CHECK-NEXT: movaz { z0.h, z1.h }, za0v.h[w12, 0:1]173; CHECK-NEXT: movaz { z0.h, z1.h }, za1v.h[w12, 6:7]174; CHECK-NEXT: ret175 %res = call {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.aarch64.sme.readz.vert.x2.nxv8bf16(i32 0, i32 %slice)176 %slice.max = add i32 %slice, 6177 %res2 = call {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.aarch64.sme.readz.vert.x2.nxv8bf16(i32 1, i32 %slice.max)178 ret {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} %res2179}180 181define {<vscale x 8 x half>, <vscale x 8 x half>} @test_readz_ver_z16_f16_x2(i32 %slice) #0 {182; CHECK-LABEL: test_readz_ver_z16_f16_x2:183; CHECK: // %bb.0:184; CHECK-NEXT: mov w12, w0185; CHECK-NEXT: movaz { z0.h, z1.h }, za0v.h[w12, 0:1]186; CHECK-NEXT: movaz { z0.h, z1.h }, za1v.h[w12, 6:7]187; CHECK-NEXT: ret188 %res = call {<vscale x 8 x half>, <vscale x 8 x half>} @llvm.aarch64.sme.readz.vert.x2.nxv8f16(i32 0, i32 %slice)189 %slice.max = add i32 %slice, 6190 %res2 = call {<vscale x 8 x half>, <vscale x 8 x half>} @llvm.aarch64.sme.readz.vert.x2.nxv8f16(i32 1, i32 %slice.max)191 ret {<vscale x 8 x half>, <vscale x 8 x half>} %res2192}193 194define {<vscale x 4 x float>, <vscale x 4 x float>} @test_readz_ver_z32_f32_x2(i32 %slice) #0 {195; CHECK-LABEL: test_readz_ver_z32_f32_x2:196; CHECK: // %bb.0:197; CHECK-NEXT: mov w12, w0198; CHECK-NEXT: movaz { z0.s, z1.s }, za0v.s[w12, 0:1]199; CHECK-NEXT: movaz { z0.s, z1.s }, za3v.s[w12, 2:3]200; CHECK-NEXT: ret201 %res = call {<vscale x 4 x float>, <vscale x 4 x float>} @llvm.aarch64.sme.readz.vert.x2.nxv4f32(i32 0, i32 %slice)202 %slice.max = add i32 %slice, 2203 %res2 = call {<vscale x 4 x float>, <vscale x 4 x float>} @llvm.aarch64.sme.readz.vert.x2.nxv4f32(i32 3, i32 %slice.max)204 ret {<vscale x 4 x float>, <vscale x 4 x float>} %res2205}206 207define {<vscale x 2 x double>, <vscale x 2 x double>} @test_readz_ver_z64_f64_x2(i32 %slice) #0 {208; CHECK-LABEL: test_readz_ver_z64_f64_x2:209; CHECK: // %bb.0:210; CHECK-NEXT: mov w12, w0211; CHECK-NEXT: movaz { z0.d, z1.d }, za0v.d[w12, 0:1]212; CHECK-NEXT: movaz { z2.d, z3.d }, za7v.d[w12, 0:1]213; CHECK-NEXT: ret214 %res = call {<vscale x 2 x double>, <vscale x 2 x double>} @llvm.aarch64.sme.readz.vert.x2.nxv2f64(i32 0, i32 %slice)215 %res2 = call {<vscale x 2 x double>, <vscale x 2 x double>} @llvm.aarch64.sme.readz.vert.x2.nxv2f64(i32 7, i32 %slice)216 ret {<vscale x 2 x double>, <vscale x 2 x double>} %res217}218 219;;220; X4 - Horiz221;;222 223define {<vscale x 16 x i8>, <vscale x 16 x i8>,<vscale x 16 x i8>, <vscale x 16 x i8>} @test_readz_hor_z8_i8_x4(i32 %tile, i32 %slice) #0 {224; CHECK-LABEL: test_readz_hor_z8_i8_x4:225; CHECK: // %bb.0:226; CHECK-NEXT: mov w12, w1227; CHECK-NEXT: movaz { z0.b - z3.b }, za0h.b[w12, 0:3]228; CHECK-NEXT: movaz { z0.b - z3.b }, za0h.b[w12, 12:15]229; CHECK-NEXT: ret230 %res = call {<vscale x 16 x i8>, <vscale x 16 x i8>,<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sme.readz.horiz.x4.nxv16i8(i32 0, i32 %slice)231 %slice.max = add i32 %slice, 12232 %res2 = call {<vscale x 16 x i8>, <vscale x 16 x i8>,<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sme.readz.horiz.x4.nxv16i8(i32 0, i32 %slice.max)233 ret {<vscale x 16 x i8>, <vscale x 16 x i8>,<vscale x 16 x i8>, <vscale x 16 x i8>} %res2234}235define {<vscale x 8 x i16>, <vscale x 8 x i16>,<vscale x 8 x i16>, <vscale x 8 x i16>} @test_readz_hor_z16_i16_x4(i32 %slice) #0 {236; CHECK-LABEL: test_readz_hor_z16_i16_x4:237; CHECK: // %bb.0:238; CHECK-NEXT: mov w12, w0239; CHECK-NEXT: movaz { z0.h - z3.h }, za0h.h[w12, 0:3]240; CHECK-NEXT: movaz { z0.h - z3.h }, za1h.h[w12, 4:7]241; CHECK-NEXT: ret242 %res = call {<vscale x 8 x i16>, <vscale x 8 x i16>,<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.aarch64.sme.readz.horiz.x4.nxv8i16(i32 0, i32 %slice)243 %slice.max = add i32 %slice, 4244 %res2 = call {<vscale x 8 x i16>, <vscale x 8 x i16>,<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.aarch64.sme.readz.horiz.x4.nxv8i16(i32 1, i32 %slice.max)245 ret {<vscale x 8 x i16>, <vscale x 8 x i16>,<vscale x 8 x i16>, <vscale x 8 x i16>} %res2246}247 248define {<vscale x 4 x i32>, <vscale x 4 x i32>,<vscale x 4 x i32>, <vscale x 4 x i32>} @test_readz_hor_z32_i32_x4(i32 %slice) #0 {249; CHECK-LABEL: test_readz_hor_z32_i32_x4:250; CHECK: // %bb.0:251; CHECK-NEXT: mov w12, w0252; CHECK-NEXT: movaz { z0.s - z3.s }, za0h.s[w12, 0:3]253; CHECK-NEXT: movaz { z0.s - z3.s }, za3h.s[w12, 0:3]254; CHECK-NEXT: ret255 %res = call {<vscale x 4 x i32>, <vscale x 4 x i32>,<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.aarch64.sme.readz.horiz.x4.nxv4i32(i32 0, i32 %slice)256 %res2 = call {<vscale x 4 x i32>, <vscale x 4 x i32>,<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.aarch64.sme.readz.horiz.x4.nxv4i32(i32 3, i32 %slice)257 ret {<vscale x 4 x i32>, <vscale x 4 x i32>,<vscale x 4 x i32>, <vscale x 4 x i32>} %res2258}259 260define {<vscale x 2 x i64>, <vscale x 2 x i64>,<vscale x 2 x i64>, <vscale x 2 x i64>} @test_readz_hor_z64_i64_x4(i32 %slice) #0 {261; CHECK-LABEL: test_readz_hor_z64_i64_x4:262; CHECK: // %bb.0:263; CHECK-NEXT: mov w12, w0264; CHECK-NEXT: movaz { z0.d - z3.d }, za0h.d[w12, 0:3]265; CHECK-NEXT: movaz { z4.d - z7.d }, za7h.d[w12, 0:3]266; CHECK-NEXT: ret267 %res = call {<vscale x 2 x i64>, <vscale x 2 x i64>,<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.aarch64.sme.readz.horiz.x4.nxv2i64(i32 0, i32 %slice)268 %res2 = call {<vscale x 2 x i64>, <vscale x 2 x i64>,<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.aarch64.sme.readz.horiz.x4.nxv2i64(i32 7, i32 %slice)269 ret {<vscale x 2 x i64>, <vscale x 2 x i64>,<vscale x 2 x i64>, <vscale x 2 x i64>} %res270}271 272define {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @test_readz_hor_z16_bf16_x4(i32 %slice) #0 {273; CHECK-LABEL: test_readz_hor_z16_bf16_x4:274; CHECK: // %bb.0:275; CHECK-NEXT: mov w12, w0276; CHECK-NEXT: movaz { z0.h - z3.h }, za0h.h[w12, 0:3]277; CHECK-NEXT: movaz { z0.h - z3.h }, za1h.h[w12, 4:7]278; CHECK-NEXT: ret279 %res = call {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.aarch64.sme.readz.horiz.x4.nxv8bf16(i32 0, i32 %slice)280 %slice.max = add i32 %slice, 4281 %res2 = call {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.aarch64.sme.readz.horiz.x4.nxv8bf16(i32 1, i32 %slice.max)282 ret {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>} %res2283}284 285define {<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>} @test_readz_hor_z16_f16_x4(i32 %slice) #0 {286; CHECK-LABEL: test_readz_hor_z16_f16_x4:287; CHECK: // %bb.0:288; CHECK-NEXT: mov w12, w0289; CHECK-NEXT: movaz { z0.h - z3.h }, za0h.h[w12, 0:3]290; CHECK-NEXT: movaz { z0.h - z3.h }, za1h.h[w12, 4:7]291; CHECK-NEXT: ret292 %res = call {<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>} @llvm.aarch64.sme.readz.horiz.x4.nxv8f16(i32 0, i32 %slice)293 %slice.max = add i32 %slice, 4294 %res2 = call {<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>} @llvm.aarch64.sme.readz.horiz.x4.nxv8f16(i32 1, i32 %slice.max)295 ret {<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>} %res2296}297 298define {<vscale x 4 x float>, <vscale x 4 x float>,<vscale x 4 x float>, <vscale x 4 x float>} @test_readz_hor_z32_f32_x4(i32 %slice) #0 {299; CHECK-LABEL: test_readz_hor_z32_f32_x4:300; CHECK: // %bb.0:301; CHECK-NEXT: mov w12, w0302; CHECK-NEXT: movaz { z0.s - z3.s }, za0h.s[w12, 0:3]303; CHECK-NEXT: movaz { z0.s - z3.s }, za3h.s[w12, 0:3]304; CHECK-NEXT: ret305 %res = call {<vscale x 4 x float>, <vscale x 4 x float>,<vscale x 4 x float>, <vscale x 4 x float>} @llvm.aarch64.sme.readz.horiz.x4.nxv4f32(i32 0, i32 %slice)306 %res2 = call {<vscale x 4 x float>, <vscale x 4 x float>,<vscale x 4 x float>, <vscale x 4 x float>} @llvm.aarch64.sme.readz.horiz.x4.nxv4f32(i32 3, i32 %slice)307 ret {<vscale x 4 x float>, <vscale x 4 x float>,<vscale x 4 x float>, <vscale x 4 x float>} %res2308}309 310define {<vscale x 2 x double>, <vscale x 2 x double>,<vscale x 2 x double>, <vscale x 2 x double>} @test_readz_hor_z64_f64_x4(i32 %slice) #0 {311; CHECK-LABEL: test_readz_hor_z64_f64_x4:312; CHECK: // %bb.0:313; CHECK-NEXT: mov w12, w0314; CHECK-NEXT: movaz { z0.d - z3.d }, za0h.d[w12, 0:3]315; CHECK-NEXT: movaz { z4.d - z7.d }, za7h.d[w12, 0:3]316; CHECK-NEXT: ret317 %res = call {<vscale x 2 x double>, <vscale x 2 x double>,<vscale x 2 x double>, <vscale x 2 x double>} @llvm.aarch64.sme.readz.horiz.x4.nxv2f64(i32 0, i32 %slice)318 %res2 = call {<vscale x 2 x double>, <vscale x 2 x double>,<vscale x 2 x double>, <vscale x 2 x double>} @llvm.aarch64.sme.readz.horiz.x4.nxv2f64(i32 7, i32 %slice)319 ret {<vscale x 2 x double>, <vscale x 2 x double>,<vscale x 2 x double>, <vscale x 2 x double>} %res320}321 322;;323; X4 - Vert324;;325 326define {<vscale x 16 x i8>, <vscale x 16 x i8>,<vscale x 16 x i8>, <vscale x 16 x i8>} @test_readz_ver_z8_i8_x4(i32 %tile, i32 %slice) #0 {327; CHECK-LABEL: test_readz_ver_z8_i8_x4:328; CHECK: // %bb.0:329; CHECK-NEXT: mov w12, w1330; CHECK-NEXT: movaz { z0.b - z3.b }, za0v.b[w12, 0:3]331; CHECK-NEXT: movaz { z0.b - z3.b }, za0v.b[w12, 12:15]332; CHECK-NEXT: ret333 %res = call {<vscale x 16 x i8>, <vscale x 16 x i8>,<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sme.readz.vert.x4.nxv16i8(i32 0, i32 %slice)334 %slice.max = add i32 %slice, 12335 %res2 = call {<vscale x 16 x i8>, <vscale x 16 x i8>,<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sme.readz.vert.x4.nxv16i8(i32 0, i32 %slice.max)336 ret {<vscale x 16 x i8>, <vscale x 16 x i8>,<vscale x 16 x i8>, <vscale x 16 x i8>} %res2337}338define {<vscale x 8 x i16>, <vscale x 8 x i16>,<vscale x 8 x i16>, <vscale x 8 x i16>} @test_readz_ver_z16_i16_x4(i32 %slice) #0 {339; CHECK-LABEL: test_readz_ver_z16_i16_x4:340; CHECK: // %bb.0:341; CHECK-NEXT: mov w12, w0342; CHECK-NEXT: movaz { z0.h - z3.h }, za0v.h[w12, 0:3]343; CHECK-NEXT: movaz { z0.h - z3.h }, za1v.h[w12, 4:7]344; CHECK-NEXT: ret345 %res = call {<vscale x 8 x i16>, <vscale x 8 x i16>,<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.aarch64.sme.readz.vert.x4.nxv8i16(i32 0, i32 %slice)346 %slice.max = add i32 %slice, 4347 %res2 = call {<vscale x 8 x i16>, <vscale x 8 x i16>,<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.aarch64.sme.readz.vert.x4.nxv8i16(i32 1, i32 %slice.max)348 ret {<vscale x 8 x i16>, <vscale x 8 x i16>,<vscale x 8 x i16>, <vscale x 8 x i16>} %res2349}350 351define {<vscale x 4 x i32>, <vscale x 4 x i32>,<vscale x 4 x i32>, <vscale x 4 x i32>} @test_readz_ver_z32_i32_x4(i32 %slice) #0 {352; CHECK-LABEL: test_readz_ver_z32_i32_x4:353; CHECK: // %bb.0:354; CHECK-NEXT: mov w12, w0355; CHECK-NEXT: movaz { z0.s - z3.s }, za0v.s[w12, 0:3]356; CHECK-NEXT: movaz { z0.s - z3.s }, za3v.s[w12, 0:3]357; CHECK-NEXT: ret358 %res = call {<vscale x 4 x i32>, <vscale x 4 x i32>,<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.aarch64.sme.readz.vert.x4.nxv4i32(i32 0, i32 %slice)359 %res2 = call {<vscale x 4 x i32>, <vscale x 4 x i32>,<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.aarch64.sme.readz.vert.x4.nxv4i32(i32 3, i32 %slice)360 ret {<vscale x 4 x i32>, <vscale x 4 x i32>,<vscale x 4 x i32>, <vscale x 4 x i32>} %res2361}362 363define {<vscale x 2 x i64>, <vscale x 2 x i64>,<vscale x 2 x i64>, <vscale x 2 x i64>} @test_readz_ver_z64_i64_x4(i32 %slice) #0 {364; CHECK-LABEL: test_readz_ver_z64_i64_x4:365; CHECK: // %bb.0:366; CHECK-NEXT: mov w12, w0367; CHECK-NEXT: movaz { z0.d - z3.d }, za0v.d[w12, 0:3]368; CHECK-NEXT: movaz { z4.d - z7.d }, za7v.d[w12, 0:3]369; CHECK-NEXT: ret370 %res = call {<vscale x 2 x i64>, <vscale x 2 x i64>,<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.aarch64.sme.readz.vert.x4.nxv2i64(i32 0, i32 %slice)371 %res2 = call {<vscale x 2 x i64>, <vscale x 2 x i64>,<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.aarch64.sme.readz.vert.x4.nxv2i64(i32 7, i32 %slice)372 ret {<vscale x 2 x i64>, <vscale x 2 x i64>,<vscale x 2 x i64>, <vscale x 2 x i64>} %res373}374 375define {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @test_readz_ver_z16_bf16_x4(i32 %slice) #0 {376; CHECK-LABEL: test_readz_ver_z16_bf16_x4:377; CHECK: // %bb.0:378; CHECK-NEXT: mov w12, w0379; CHECK-NEXT: movaz { z0.h - z3.h }, za0v.h[w12, 0:3]380; CHECK-NEXT: movaz { z0.h - z3.h }, za1v.h[w12, 4:7]381; CHECK-NEXT: ret382 %res = call {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.aarch64.sme.readz.vert.x4.nxv8bf16(i32 0, i32 %slice)383 %slice.max = add i32 %slice, 4384 %res2 = call {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.aarch64.sme.readz.vert.x4.nxv8bf16(i32 1, i32 %slice.max)385 ret {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>} %res2386}387 388define {<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>} @test_readz_ver_z16_f16_x4(i32 %slice) #0 {389; CHECK-LABEL: test_readz_ver_z16_f16_x4:390; CHECK: // %bb.0:391; CHECK-NEXT: mov w12, w0392; CHECK-NEXT: movaz { z0.h - z3.h }, za0v.h[w12, 0:3]393; CHECK-NEXT: movaz { z0.h - z3.h }, za1v.h[w12, 4:7]394; CHECK-NEXT: ret395 %res = call {<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>} @llvm.aarch64.sme.readz.vert.x4.nxv8f16(i32 0, i32 %slice)396 %slice.max = add i32 %slice, 4397 %res2 = call {<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>} @llvm.aarch64.sme.readz.vert.x4.nxv8f16(i32 1, i32 %slice.max)398 ret {<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>} %res2399}400 401define {<vscale x 4 x float>, <vscale x 4 x float>,<vscale x 4 x float>, <vscale x 4 x float>} @test_readz_ver_z32_f32_x4(i32 %slice) #0 {402; CHECK-LABEL: test_readz_ver_z32_f32_x4:403; CHECK: // %bb.0:404; CHECK-NEXT: mov w12, w0405; CHECK-NEXT: movaz { z0.s - z3.s }, za0v.s[w12, 0:3]406; CHECK-NEXT: movaz { z0.s - z3.s }, za3v.s[w12, 0:3]407; CHECK-NEXT: ret408 %res = call {<vscale x 4 x float>, <vscale x 4 x float>,<vscale x 4 x float>, <vscale x 4 x float>} @llvm.aarch64.sme.readz.vert.x4.nxv4f32(i32 0, i32 %slice)409 %res2 = call {<vscale x 4 x float>, <vscale x 4 x float>,<vscale x 4 x float>, <vscale x 4 x float>} @llvm.aarch64.sme.readz.vert.x4.nxv4f32(i32 3, i32 %slice)410 ret {<vscale x 4 x float>, <vscale x 4 x float>,<vscale x 4 x float>, <vscale x 4 x float>} %res2411}412 413define {<vscale x 2 x double>, <vscale x 2 x double>,<vscale x 2 x double>, <vscale x 2 x double>} @test_readz_ver_z64_f64_x4(i32 %slice) #0 {414; CHECK-LABEL: test_readz_ver_z64_f64_x4:415; CHECK: // %bb.0:416; CHECK-NEXT: mov w12, w0417; CHECK-NEXT: movaz { z0.d - z3.d }, za0v.d[w12, 0:3]418; CHECK-NEXT: movaz { z4.d - z7.d }, za7v.d[w12, 0:3]419; CHECK-NEXT: ret420 %res = call {<vscale x 2 x double>, <vscale x 2 x double>,<vscale x 2 x double>, <vscale x 2 x double>} @llvm.aarch64.sme.readz.vert.x4.nxv2f64(i32 0, i32 %slice)421 %res2 = call {<vscale x 2 x double>, <vscale x 2 x double>,<vscale x 2 x double>, <vscale x 2 x double>} @llvm.aarch64.sme.readz.vert.x4.nxv2f64(i32 7, i32 %slice)422 ret {<vscale x 2 x double>, <vscale x 2 x double>,<vscale x 2 x double>, <vscale x 2 x double>} %res423}424 425 426declare {<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sme.readz.horiz.za8.x2.nxv16i8(i32, i32)427declare {<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.aarch64.sme.readz.horiz.x2.nxv8i16(i32, i32)428declare {<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.aarch64.sme.readz.horiz.x2.nxv4i32(i32, i32)429declare {<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.aarch64.sme.readz.horiz.x2.nxv2i64(i32, i32)430declare {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.aarch64.sme.readz.horiz.x2.nxv8bf16(i32, i32)431declare {<vscale x 8 x half>, <vscale x 8 x half>} @llvm.aarch64.sme.readz.horiz.x2.nxv8f16(i32, i32)432declare {<vscale x 4 x float>, <vscale x 4 x float>} @llvm.aarch64.sme.readz.horiz.x2.nxv4f32(i32, i32)433declare {<vscale x 2 x double>, <vscale x 2 x double>} @llvm.aarch64.sme.readz.horiz.x2.nxv2f64(i32, i32)434 435declare {<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sme.readz.vert.za8.x2.nxv16i8(i32, i32)436declare {<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.aarch64.sme.readz.vert.x2.nxv8i16(i32, i32)437declare {<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.aarch64.sme.readz.vert.x2.nxv4i32(i32, i32)438declare {<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.aarch64.sme.readz.vert.x2.nxv2i64(i32, i32)439declare {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.aarch64.sme.readz.vert.x2.nxv8bf16(i32, i32)440declare {<vscale x 8 x half>, <vscale x 8 x half>} @llvm.aarch64.sme.readz.vert.x2.nxv8f16(i32, i32)441declare {<vscale x 4 x float>, <vscale x 4 x float>} @llvm.aarch64.sme.readz.vert.x2.nxv4f32(i32, i32)442declare {<vscale x 2 x double>, <vscale x 2 x double>} @llvm.aarch64.sme.readz.vert.x2.nxv2f64(i32, i32)443 444declare {<vscale x 16 x i8>, <vscale x 16 x i8>,<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sme.readz.horiz.za8.x4.nxv16i8(i32, i32)445declare {<vscale x 8 x i16>, <vscale x 8 x i16>,<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.aarch64.sme.readz.horiz.x4.nxv8i16(i32, i32)446declare {<vscale x 4 x i32>, <vscale x 4 x i32>,<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.aarch64.sme.readz.horiz.x4.nxv4i32(i32, i32)447declare {<vscale x 2 x i64>, <vscale x 2 x i64>,<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.aarch64.sme.readz.horiz.x4.nxv2i64(i32, i32)448declare {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.aarch64.sme.readz.horiz.x4.nxv8bf16(i32, i32)449declare {<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>} @llvm.aarch64.sme.readz.horiz.x4.nxv8f16(i32, i32)450declare {<vscale x 4 x float>, <vscale x 4 x float>,<vscale x 4 x float>, <vscale x 4 x float>} @llvm.aarch64.sme.readz.horiz.x4.nxv4f32(i32, i32)451declare {<vscale x 2 x double>, <vscale x 2 x double>,<vscale x 2 x double>, <vscale x 2 x double>} @llvm.aarch64.sme.readz.horiz.x4.nxv2f64(i32, i32)452 453declare {<vscale x 16 x i8>, <vscale x 16 x i8>,<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sme.readz.vert.za8.x4.nxv16i8(i32, i32)454declare {<vscale x 8 x i16>, <vscale x 8 x i16>,<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.aarch64.sme.readz.vert.x4.nxv8i16(i32, i32)455declare {<vscale x 4 x i32>, <vscale x 4 x i32>,<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.aarch64.sme.readz.vert.x4.nxv4i32(i32, i32)456declare {<vscale x 2 x i64>, <vscale x 2 x i64>,<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.aarch64.sme.readz.vert.x4.nxv2i64(i32, i32)457declare {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.aarch64.sme.readz.vert.x4.nxv8bf16(i32, i32)458declare {<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>} @llvm.aarch64.sme.readz.vert.x4.nxv8f16(i32, i32)459declare {<vscale x 4 x float>, <vscale x 4 x float>,<vscale x 4 x float>, <vscale x 4 x float>} @llvm.aarch64.sme.readz.vert.x4.nxv4f32(i32, i32)460declare {<vscale x 2 x double>, <vscale x 2 x double>,<vscale x 2 x double>, <vscale x 2 x double>} @llvm.aarch64.sme.readz.vert.x4.nxv2f64(i32, i32)461 462;MOVAZ (tile to vector, single)463 464;;465; Horiz466;;467define <vscale x 16 x i8> @test_readz_hor_z8_i8(i32 %tile, i32 %slice) #0 {468; CHECK-LABEL: test_readz_hor_z8_i8:469; CHECK: // %bb.0:470; CHECK-NEXT: mov w12, w1471; CHECK-NEXT: movaz z0.b, za0h.b[w12, 0]472; CHECK-NEXT: movaz z0.b, za0h.b[w12, 14]473; CHECK-NEXT: ret474 %res = call <vscale x 16 x i8> @llvm.aarch64.sme.readz.horiz.nxv16i8(i32 0, i32 %slice)475 %slice.max = add i32 %slice, 14476 %res2 = call <vscale x 16 x i8> @llvm.aarch64.sme.readz.horiz.nxv16i8(i32 0, i32 %slice.max)477 ret <vscale x 16 x i8> %res2478}479 480define <vscale x 8 x i16> @test_readz_hor_z16_i16(i32 %tile, i32 %slice) #0 {481; CHECK-LABEL: test_readz_hor_z16_i16:482; CHECK: // %bb.0:483; CHECK-NEXT: mov w12, w1484; CHECK-NEXT: movaz z0.h, za0h.h[w12, 0]485; CHECK-NEXT: movaz z0.h, za1h.h[w12, 7]486; CHECK-NEXT: ret487 %res = call <vscale x 8 x i16> @llvm.aarch64.sme.readz.horiz.nxv8i16(i32 0, i32 %slice)488 %slice.max = add i32 %slice, 7489 %res2 = call <vscale x 8 x i16> @llvm.aarch64.sme.readz.horiz.nxv8i16(i32 1, i32 %slice.max)490 ret <vscale x 8 x i16> %res2491}492 493define <vscale x 4 x i32> @test_readz_hor_z32_i32(i32 %tile, i32 %slice) #0 {494; CHECK-LABEL: test_readz_hor_z32_i32:495; CHECK: // %bb.0:496; CHECK-NEXT: mov w12, w1497; CHECK-NEXT: movaz z0.s, za0h.s[w12, 0]498; CHECK-NEXT: movaz z0.s, za3h.s[w12, 3]499; CHECK-NEXT: ret500 %res = call <vscale x 4 x i32> @llvm.aarch64.sme.readz.horiz.nxv4i32(i32 0, i32 %slice)501 %slice.max = add i32 %slice, 3502 %res2 = call <vscale x 4 x i32> @llvm.aarch64.sme.readz.horiz.nxv4i32(i32 3, i32 %slice.max)503 ret <vscale x 4 x i32> %res2504}505 506define <vscale x 2 x i64> @test_readz_hor_z64_i64(i32 %tile, i32 %slice) #0 {507; CHECK-LABEL: test_readz_hor_z64_i64:508; CHECK: // %bb.0:509; CHECK-NEXT: mov w12, w1510; CHECK-NEXT: movaz z0.d, za0h.d[w12, 0]511; CHECK-NEXT: movaz z1.d, za7h.d[w12, 1]512; CHECK-NEXT: ret513 %res = call <vscale x 2 x i64> @llvm.aarch64.sme.readz.horiz.nxv2i64(i32 0, i32 %slice)514 %slice.max = add i32 %slice, 1515 %res2 = call <vscale x 2 x i64> @llvm.aarch64.sme.readz.horiz.nxv2i64(i32 7, i32 %slice.max)516 ret <vscale x 2 x i64> %res517}518 519define <vscale x 8 x bfloat> @test_readz_hor_z16_bf16(i32 %tile, i32 %slice) #0 {520; CHECK-LABEL: test_readz_hor_z16_bf16:521; CHECK: // %bb.0:522; CHECK-NEXT: mov w12, w1523; CHECK-NEXT: movaz z0.h, za0h.h[w12, 0]524; CHECK-NEXT: movaz z0.h, za1h.h[w12, 7]525; CHECK-NEXT: ret526 %res = call <vscale x 8 x bfloat> @llvm.aarch64.sme.readz.horiz.nxv8bf16(i32 0, i32 %slice)527 %slice.max = add i32 %slice, 7528 %res2 = call <vscale x 8 x bfloat> @llvm.aarch64.sme.readz.horiz.nxv8bf16(i32 1, i32 %slice.max)529 ret <vscale x 8 x bfloat> %res2530}531 532define <vscale x 8 x half> @test_readz_hor_z16_f16(i32 %tile, i32 %slice) #0 {533; CHECK-LABEL: test_readz_hor_z16_f16:534; CHECK: // %bb.0:535; CHECK-NEXT: mov w12, w1536; CHECK-NEXT: movaz z0.h, za0h.h[w12, 0]537; CHECK-NEXT: movaz z0.h, za1h.h[w12, 7]538; CHECK-NEXT: ret539 %res = call <vscale x 8 x half> @llvm.aarch64.sme.readz.horiz.nxv8f16(i32 0, i32 %slice)540 %slice.max = add i32 %slice, 7541 %res2 = call <vscale x 8 x half> @llvm.aarch64.sme.readz.horiz.nxv8f16(i32 1, i32 %slice.max)542 ret <vscale x 8 x half> %res2543}544 545define <vscale x 4 x float> @test_readz_hor_z32_f32(i32 %tile, i32 %slice) #0 {546; CHECK-LABEL: test_readz_hor_z32_f32:547; CHECK: // %bb.0:548; CHECK-NEXT: mov w12, w1549; CHECK-NEXT: movaz z0.s, za0h.s[w12, 0]550; CHECK-NEXT: movaz z0.s, za3h.s[w12, 3]551; CHECK-NEXT: ret552 %res = call <vscale x 4 x float> @llvm.aarch64.sme.readz.horiz.nxv4f32(i32 0, i32 %slice)553 %slice.max = add i32 %slice, 3554 %res2 = call <vscale x 4 x float> @llvm.aarch64.sme.readz.horiz.nxv4f32(i32 3, i32 %slice.max)555 ret <vscale x 4 x float> %res2556}557 558define <vscale x 2 x double> @test_readz_hor_z64_f64(i32 %tile, i32 %slice) #0 {559; CHECK-LABEL: test_readz_hor_z64_f64:560; CHECK: // %bb.0:561; CHECK-NEXT: mov w12, w1562; CHECK-NEXT: movaz z0.d, za0h.d[w12, 0]563; CHECK-NEXT: movaz z1.d, za7h.d[w12, 1]564; CHECK-NEXT: ret565 %res = call <vscale x 2 x double> @llvm.aarch64.sme.readz.horiz.nxv2f64(i32 0, i32 %slice)566 %slice.max = add i32 %slice, 1567 %res2 = call <vscale x 2 x double> @llvm.aarch64.sme.readz.horiz.nxv2f64(i32 7, i32 %slice.max)568 ret <vscale x 2 x double> %res569}570 571define <vscale x 16 x i8> @test_readz_hor_z128_i8(i32 %tile, i32 %slice) #0 {572; CHECK-LABEL: test_readz_hor_z128_i8:573; CHECK: // %bb.0:574; CHECK-NEXT: mov w12, w1575; CHECK-NEXT: movaz z0.q, za0h.q[w12, 0]576; CHECK-NEXT: movaz z0.q, za15h.q[w12, 0]577; CHECK-NEXT: ret578 %res = call <vscale x 16 x i8> @llvm.aarch64.sme.readz.q.horiz.nxv16i8(i32 0, i32 %slice)579 %res2 = call <vscale x 16 x i8> @llvm.aarch64.sme.readz.q.horiz.nxv16i8(i32 15, i32 %slice)580 ret <vscale x 16 x i8> %res2581}582 583define <vscale x 8 x i16> @test_readz_hor_z128_i16(i32 %tile, i32 %slice) #0 {584; CHECK-LABEL: test_readz_hor_z128_i16:585; CHECK: // %bb.0:586; CHECK-NEXT: mov w12, w1587; CHECK-NEXT: movaz z0.q, za0h.q[w12, 0]588; CHECK-NEXT: movaz z0.q, za15h.q[w12, 0]589; CHECK-NEXT: ret590 %res = call <vscale x 8 x i16> @llvm.aarch64.sme.readz.q.horiz.nxv8i16(i32 0, i32 %slice)591 %res2 = call <vscale x 8 x i16> @llvm.aarch64.sme.readz.q.horiz.nxv8i16(i32 15, i32 %slice)592 ret <vscale x 8 x i16> %res2593}594 595define <vscale x 4 x i32> @test_readz_hor_z128_i32(i32 %tile, i32 %slice) #0 {596; CHECK-LABEL: test_readz_hor_z128_i32:597; CHECK: // %bb.0:598; CHECK-NEXT: mov w12, w1599; CHECK-NEXT: movaz z0.q, za0h.q[w12, 0]600; CHECK-NEXT: movaz z0.q, za15h.q[w12, 0]601; CHECK-NEXT: ret602 %res = call <vscale x 4 x i32> @llvm.aarch64.sme.readz.q.horiz.nxv4i32(i32 0, i32 %slice)603 %res2 = call <vscale x 4 x i32> @llvm.aarch64.sme.readz.q.horiz.nxv4i32(i32 15, i32 %slice)604 ret <vscale x 4 x i32> %res2605}606 607define <vscale x 2 x i64> @test_readz_hor_z128_i64(i32 %tile, i32 %slice) #0 {608; CHECK-LABEL: test_readz_hor_z128_i64:609; CHECK: // %bb.0:610; CHECK-NEXT: mov w12, w1611; CHECK-NEXT: movaz z0.q, za0h.q[w12, 0]612; CHECK-NEXT: movaz z1.q, za15h.q[w12, 0]613; CHECK-NEXT: ret614 %res = call <vscale x 2 x i64> @llvm.aarch64.sme.readz.q.horiz.nxv2i64(i32 0, i32 %slice)615 %res2 = call <vscale x 2 x i64> @llvm.aarch64.sme.readz.q.horiz.nxv2i64(i32 15, i32 %slice)616 ret <vscale x 2 x i64> %res617}618 619define <vscale x 8 x bfloat> @test_readz_hor_z128_bf16(i32 %tile, i32 %slice) #0 {620; CHECK-LABEL: test_readz_hor_z128_bf16:621; CHECK: // %bb.0:622; CHECK-NEXT: mov w12, w1623; CHECK-NEXT: movaz z0.q, za0h.q[w12, 0]624; CHECK-NEXT: movaz z0.q, za15h.q[w12, 0]625; CHECK-NEXT: ret626 %res = call <vscale x 8 x bfloat> @llvm.aarch64.sme.readz.q.horiz.nxv8bf16(i32 0, i32 %slice)627 %res2 = call <vscale x 8 x bfloat> @llvm.aarch64.sme.readz.q.horiz.nxv8bf16(i32 15, i32 %slice)628 ret <vscale x 8 x bfloat> %res2629}630 631define <vscale x 8 x half> @test_readz_hor_z128_f16(i32 %tile, i32 %slice) #0 {632; CHECK-LABEL: test_readz_hor_z128_f16:633; CHECK: // %bb.0:634; CHECK-NEXT: mov w12, w1635; CHECK-NEXT: movaz z0.q, za0h.q[w12, 0]636; CHECK-NEXT: movaz z0.q, za15h.q[w12, 0]637; CHECK-NEXT: ret638 %res = call <vscale x 8 x half> @llvm.aarch64.sme.readz.q.horiz.nxv8f16(i32 0, i32 %slice)639 %res2 = call <vscale x 8 x half> @llvm.aarch64.sme.readz.q.horiz.nxv8f16(i32 15, i32 %slice)640 ret <vscale x 8 x half> %res2641}642 643define <vscale x 4 x float> @test_readz_hor_z128_f32(i32 %tile, i32 %slice) #0 {644; CHECK-LABEL: test_readz_hor_z128_f32:645; CHECK: // %bb.0:646; CHECK-NEXT: mov w12, w1647; CHECK-NEXT: movaz z0.q, za0h.q[w12, 0]648; CHECK-NEXT: movaz z0.q, za15h.q[w12, 0]649; CHECK-NEXT: ret650 %res = call <vscale x 4 x float> @llvm.aarch64.sme.readz.q.horiz.nxv4f32(i32 0, i32 %slice)651 %res2 = call <vscale x 4 x float> @llvm.aarch64.sme.readz.q.horiz.nxv4f32(i32 15, i32 %slice)652 ret <vscale x 4 x float> %res2653}654 655define <vscale x 2 x double> @test_readz_hor_z128_f64(i32 %tile, i32 %slice) #0 {656; CHECK-LABEL: test_readz_hor_z128_f64:657; CHECK: // %bb.0:658; CHECK-NEXT: mov w12, w1659; CHECK-NEXT: movaz z0.q, za0h.q[w12, 0]660; CHECK-NEXT: movaz z1.q, za15h.q[w12, 0]661; CHECK-NEXT: ret662 %res = call <vscale x 2 x double> @llvm.aarch64.sme.readz.q.horiz.nxv2f64(i32 0, i32 %slice)663 %res2 = call <vscale x 2 x double> @llvm.aarch64.sme.readz.q.horiz.nxv2f64(i32 15, i32 %slice)664 ret <vscale x 2 x double> %res665}666 667;;668; Vert669;;670define <vscale x 16 x i8> @test_readz_ver_z8_i8(i32 %tile, i32 %slice) #0 {671; CHECK-LABEL: test_readz_ver_z8_i8:672; CHECK: // %bb.0:673; CHECK-NEXT: mov w12, w1674; CHECK-NEXT: movaz z0.b, za0v.b[w12, 0]675; CHECK-NEXT: movaz z0.b, za0v.b[w12, 14]676; CHECK-NEXT: ret677 %res = call <vscale x 16 x i8> @llvm.aarch64.sme.readz.vert.nxv16i8(i32 0, i32 %slice)678 %slice.max = add i32 %slice, 14679 %res2 = call <vscale x 16 x i8> @llvm.aarch64.sme.readz.vert.nxv16i8(i32 0, i32 %slice.max)680 ret <vscale x 16 x i8> %res2681}682 683define <vscale x 8 x i16> @test_readz_ver_z16_i16(i32 %tile, i32 %slice) #0 {684; CHECK-LABEL: test_readz_ver_z16_i16:685; CHECK: // %bb.0:686; CHECK-NEXT: mov w12, w1687; CHECK-NEXT: movaz z0.h, za0v.h[w12, 0]688; CHECK-NEXT: movaz z0.h, za1v.h[w12, 7]689; CHECK-NEXT: ret690 %res = call <vscale x 8 x i16> @llvm.aarch64.sme.readz.vert.nxv8i16(i32 0, i32 %slice)691 %slice.max = add i32 %slice, 7692 %res2 = call <vscale x 8 x i16> @llvm.aarch64.sme.readz.vert.nxv8i16(i32 1, i32 %slice.max)693 ret <vscale x 8 x i16> %res2694}695 696define <vscale x 4 x i32> @test_readz_ver_z32_i32(i32 %tile, i32 %slice) #0 {697; CHECK-LABEL: test_readz_ver_z32_i32:698; CHECK: // %bb.0:699; CHECK-NEXT: mov w12, w1700; CHECK-NEXT: movaz z0.s, za0v.s[w12, 0]701; CHECK-NEXT: movaz z0.s, za3v.s[w12, 3]702; CHECK-NEXT: ret703 %res = call <vscale x 4 x i32> @llvm.aarch64.sme.readz.vert.nxv4i32(i32 0, i32 %slice)704 %slice.max = add i32 %slice, 3705 %res2 = call <vscale x 4 x i32> @llvm.aarch64.sme.readz.vert.nxv4i32(i32 3, i32 %slice.max)706 ret <vscale x 4 x i32> %res2707}708 709define <vscale x 2 x i64> @test_readz_ver_z64_i64(i32 %tile, i32 %slice) #0 {710; CHECK-LABEL: test_readz_ver_z64_i64:711; CHECK: // %bb.0:712; CHECK-NEXT: mov w12, w1713; CHECK-NEXT: movaz z0.d, za0v.d[w12, 0]714; CHECK-NEXT: movaz z1.d, za7v.d[w12, 1]715; CHECK-NEXT: ret716 %res = call <vscale x 2 x i64> @llvm.aarch64.sme.readz.vert.nxv2i64(i32 0, i32 %slice)717 %slice.max = add i32 %slice, 1718 %res2 = call <vscale x 2 x i64> @llvm.aarch64.sme.readz.vert.nxv2i64(i32 7, i32 %slice.max)719 ret <vscale x 2 x i64> %res720}721 722define <vscale x 8 x bfloat> @test_readz_ver_z16_bf16(i32 %tile, i32 %slice) #0 {723; CHECK-LABEL: test_readz_ver_z16_bf16:724; CHECK: // %bb.0:725; CHECK-NEXT: mov w12, w1726; CHECK-NEXT: movaz z0.h, za0v.h[w12, 0]727; CHECK-NEXT: movaz z0.h, za1v.h[w12, 7]728; CHECK-NEXT: ret729 %res = call <vscale x 8 x bfloat> @llvm.aarch64.sme.readz.vert.nxv8bf16(i32 0, i32 %slice)730 %slice.max = add i32 %slice, 7731 %res2 = call <vscale x 8 x bfloat> @llvm.aarch64.sme.readz.vert.nxv8bf16(i32 1, i32 %slice.max)732 ret <vscale x 8 x bfloat> %res2733}734 735define <vscale x 8 x half> @test_readz_ver_z16_f16(i32 %tile, i32 %slice) #0 {736; CHECK-LABEL: test_readz_ver_z16_f16:737; CHECK: // %bb.0:738; CHECK-NEXT: mov w12, w1739; CHECK-NEXT: movaz z0.h, za0v.h[w12, 0]740; CHECK-NEXT: movaz z0.h, za1v.h[w12, 7]741; CHECK-NEXT: ret742 %res = call <vscale x 8 x half> @llvm.aarch64.sme.readz.vert.nxv8f16(i32 0, i32 %slice)743 %slice.max = add i32 %slice, 7744 %res2 = call <vscale x 8 x half> @llvm.aarch64.sme.readz.vert.nxv8f16(i32 1, i32 %slice.max)745 ret <vscale x 8 x half> %res2746}747 748define <vscale x 4 x float> @test_readz_ver_z32_f32(i32 %tile, i32 %slice) #0 {749; CHECK-LABEL: test_readz_ver_z32_f32:750; CHECK: // %bb.0:751; CHECK-NEXT: mov w12, w1752; CHECK-NEXT: movaz z0.s, za0v.s[w12, 0]753; CHECK-NEXT: movaz z0.s, za3v.s[w12, 3]754; CHECK-NEXT: ret755 %res = call <vscale x 4 x float> @llvm.aarch64.sme.readz.vert.nxv4f32(i32 0, i32 %slice)756 %slice.max = add i32 %slice, 3757 %res2 = call <vscale x 4 x float> @llvm.aarch64.sme.readz.vert.nxv4f32(i32 3, i32 %slice.max)758 ret <vscale x 4 x float> %res2759}760 761define <vscale x 2 x double> @test_readz_ver_z64_f64(i32 %tile, i32 %slice) #0 {762; CHECK-LABEL: test_readz_ver_z64_f64:763; CHECK: // %bb.0:764; CHECK-NEXT: mov w12, w1765; CHECK-NEXT: movaz z0.d, za0v.d[w12, 0]766; CHECK-NEXT: movaz z1.d, za7v.d[w12, 1]767; CHECK-NEXT: ret768 %res = call <vscale x 2 x double> @llvm.aarch64.sme.readz.vert.nxv2f64(i32 0, i32 %slice)769 %slice.max = add i32 %slice, 1770 %res2 = call <vscale x 2 x double> @llvm.aarch64.sme.readz.vert.nxv2f64(i32 7, i32 %slice.max)771 ret <vscale x 2 x double> %res772}773 774define <vscale x 16 x i8> @test_readz_ver_z128_i8(i32 %tile, i32 %slice) #0 {775; CHECK-LABEL: test_readz_ver_z128_i8:776; CHECK: // %bb.0:777; CHECK-NEXT: mov w12, w1778; CHECK-NEXT: movaz z0.q, za0v.q[w12, 0]779; CHECK-NEXT: movaz z0.q, za15v.q[w12, 0]780; CHECK-NEXT: ret781 %res = call <vscale x 16 x i8> @llvm.aarch64.sme.readz.q.vert.nxv16i8(i32 0, i32 %slice)782 %res2 = call <vscale x 16 x i8> @llvm.aarch64.sme.readz.q.vert.nxv16i8(i32 15, i32 %slice)783 ret <vscale x 16 x i8> %res2784}785 786define <vscale x 8 x i16> @test_readz_ver_z128_i16(i32 %tile, i32 %slice) #0 {787; CHECK-LABEL: test_readz_ver_z128_i16:788; CHECK: // %bb.0:789; CHECK-NEXT: mov w12, w1790; CHECK-NEXT: movaz z0.q, za0v.q[w12, 0]791; CHECK-NEXT: movaz z0.q, za15v.q[w12, 0]792; CHECK-NEXT: ret793 %res = call <vscale x 8 x i16> @llvm.aarch64.sme.readz.q.vert.nxv8i16(i32 0, i32 %slice)794 %res2 = call <vscale x 8 x i16> @llvm.aarch64.sme.readz.q.vert.nxv8i16(i32 15, i32 %slice)795 ret <vscale x 8 x i16> %res2796}797 798define <vscale x 4 x i32> @test_readz_ver_z128_i32(i32 %tile, i32 %slice) #0 {799; CHECK-LABEL: test_readz_ver_z128_i32:800; CHECK: // %bb.0:801; CHECK-NEXT: mov w12, w1802; CHECK-NEXT: movaz z0.q, za0v.q[w12, 0]803; CHECK-NEXT: movaz z0.q, za15v.q[w12, 0]804; CHECK-NEXT: ret805 %res = call <vscale x 4 x i32> @llvm.aarch64.sme.readz.q.vert.nxv4i32(i32 0, i32 %slice)806 %res2 = call <vscale x 4 x i32> @llvm.aarch64.sme.readz.q.vert.nxv4i32(i32 15, i32 %slice)807 ret <vscale x 4 x i32> %res2808}809 810define <vscale x 2 x i64> @test_readz_ver_z128_i64(i32 %tile, i32 %slice) #0 {811; CHECK-LABEL: test_readz_ver_z128_i64:812; CHECK: // %bb.0:813; CHECK-NEXT: mov w12, w1814; CHECK-NEXT: movaz z0.q, za0v.q[w12, 0]815; CHECK-NEXT: movaz z1.q, za15v.q[w12, 0]816; CHECK-NEXT: ret817 %res = call <vscale x 2 x i64> @llvm.aarch64.sme.readz.q.vert.nxv2i64(i32 0, i32 %slice)818 %res2 = call <vscale x 2 x i64> @llvm.aarch64.sme.readz.q.vert.nxv2i64(i32 15, i32 %slice)819 ret <vscale x 2 x i64> %res820}821 822define <vscale x 8 x bfloat> @test_readz_ver_z128_bf16(i32 %tile, i32 %slice) #0 {823; CHECK-LABEL: test_readz_ver_z128_bf16:824; CHECK: // %bb.0:825; CHECK-NEXT: mov w12, w1826; CHECK-NEXT: movaz z0.q, za0v.q[w12, 0]827; CHECK-NEXT: movaz z0.q, za15v.q[w12, 0]828; CHECK-NEXT: ret829 %res = call <vscale x 8 x bfloat> @llvm.aarch64.sme.readz.q.vert.nxv8bf16(i32 0, i32 %slice)830 %res2 = call <vscale x 8 x bfloat> @llvm.aarch64.sme.readz.q.vert.nxv8bf16(i32 15, i32 %slice)831 ret <vscale x 8 x bfloat> %res2832}833 834define <vscale x 8 x half> @test_readz_ver_z128_f16(i32 %tile, i32 %slice) #0 {835; CHECK-LABEL: test_readz_ver_z128_f16:836; CHECK: // %bb.0:837; CHECK-NEXT: mov w12, w1838; CHECK-NEXT: movaz z0.q, za0v.q[w12, 0]839; CHECK-NEXT: movaz z0.q, za15v.q[w12, 0]840; CHECK-NEXT: ret841 %res = call <vscale x 8 x half> @llvm.aarch64.sme.readz.q.vert.nxv8f16(i32 0, i32 %slice)842 %res2 = call <vscale x 8 x half> @llvm.aarch64.sme.readz.q.vert.nxv8f16(i32 15, i32 %slice)843 ret <vscale x 8 x half> %res2844}845 846define <vscale x 4 x float> @test_readz_ver_z128_f32(i32 %tile, i32 %slice) #0 {847; CHECK-LABEL: test_readz_ver_z128_f32:848; CHECK: // %bb.0:849; CHECK-NEXT: mov w12, w1850; CHECK-NEXT: movaz z0.q, za0v.q[w12, 0]851; CHECK-NEXT: movaz z0.q, za15v.q[w12, 0]852; CHECK-NEXT: ret853 %res = call <vscale x 4 x float> @llvm.aarch64.sme.readz.q.vert.nxv4f32(i32 0, i32 %slice)854 %res2 = call <vscale x 4 x float> @llvm.aarch64.sme.readz.q.vert.nxv4f32(i32 15, i32 %slice)855 ret <vscale x 4 x float> %res2856}857 858define <vscale x 2 x double> @test_readz_ver_z128_f64(i32 %tile, i32 %slice) #0 {859; CHECK-LABEL: test_readz_ver_z128_f64:860; CHECK: // %bb.0:861; CHECK-NEXT: mov w12, w1862; CHECK-NEXT: movaz z0.q, za0v.q[w12, 0]863; CHECK-NEXT: movaz z1.q, za15v.q[w12, 0]864; CHECK-NEXT: ret865 %res = call <vscale x 2 x double> @llvm.aarch64.sme.readz.q.vert.nxv2f64(i32 0, i32 %slice)866 %res2 = call <vscale x 2 x double> @llvm.aarch64.sme.readz.q.vert.nxv2f64(i32 15, i32 %slice)867 ret <vscale x 2 x double> %res868}869 870declare <vscale x 16 x i8> @llvm.aarch64.sme.readz.horiz.nxv16i8(i32, i32)871declare <vscale x 8 x i16> @llvm.aarch64.sme.readz.horiz.nxv8i16(i32, i32)872declare <vscale x 4 x i32> @llvm.aarch64.sme.readz.horiz.nxv4i32(i32, i32)873declare <vscale x 2 x i64> @llvm.aarch64.sme.readz.horiz.nxv2i64(i32, i32)874declare <vscale x 8 x bfloat> @llvm.aarch64.sme.readz.horiz.nxv8bf16(i32, i32)875declare <vscale x 8 x half> @llvm.aarch64.sme.readz.horiz.nxv8f16(i32, i32)876declare <vscale x 4 x float> @llvm.aarch64.sme.readz.horiz.nxv4f32(i32, i32)877declare <vscale x 2 x double> @llvm.aarch64.sme.readz.horiz.nxv2f64(i32, i32)878declare <vscale x 16 x i8> @llvm.aarch64.sme.readz.q.horiz.nxv16i8(i32, i32)879declare <vscale x 8 x i16> @llvm.aarch64.sme.readz.q.horiz.nxv8i16(i32, i32)880declare <vscale x 4 x i32> @llvm.aarch64.sme.readz.q.horiz.nxv4i32(i32, i32)881declare <vscale x 2 x i64> @llvm.aarch64.sme.readz.q.horiz.nxv2i64(i32, i32)882declare <vscale x 8 x bfloat> @llvm.aarch64.sme.readz.q.horiz.nxv8bf16(i32, i32)883declare <vscale x 8 x half> @llvm.aarch64.sme.readz.q.horiz.nxv8f16(i32, i32)884declare <vscale x 4 x float> @llvm.aarch64.sme.readz.q.horiz.nxv4f32(i32, i32)885declare <vscale x 2 x double> @llvm.aarch64.sme.readz.q.horiz.nxv2f64(i32, i32)886 887 888declare <vscale x 16 x i8> @llvm.aarch64.sme.readz.vert.nxv16i8(i32, i32)889declare <vscale x 8 x i16> @llvm.aarch64.sme.readz.vert.nxv8i16(i32, i32)890declare <vscale x 4 x i32> @llvm.aarch64.sme.readz.vert.nxv4i32(i32, i32)891declare <vscale x 2 x i64> @llvm.aarch64.sme.readz.vert.nxv2i64(i32, i32)892declare <vscale x 8 x bfloat> @llvm.aarch64.sme.readz.vert.nxv8bf16(i32, i32)893declare <vscale x 8 x half> @llvm.aarch64.sme.readz.vert.nxv8f16(i32, i32)894declare <vscale x 4 x float> @llvm.aarch64.sme.readz.vert.nxv4f32(i32, i32)895declare <vscale x 2 x double> @llvm.aarch64.sme.readz.vert.nxv2f64(i32, i32)896declare <vscale x 16 x i8> @llvm.aarch64.sme.readz.q.vert.nxv16i8(i32, i32)897declare <vscale x 8 x i16> @llvm.aarch64.sme.readz.q.vert.nxv8i16(i32, i32)898declare <vscale x 4 x i32> @llvm.aarch64.sme.readz.q.vert.nxv4i32(i32, i32)899declare <vscale x 2 x i64> @llvm.aarch64.sme.readz.q.vert.nxv2i64(i32, i32)900declare <vscale x 8 x bfloat> @llvm.aarch64.sme.readz.q.vert.nxv8bf16(i32, i32)901declare <vscale x 8 x half> @llvm.aarch64.sme.readz.q.vert.nxv8f16(i32, i32)902declare <vscale x 4 x float> @llvm.aarch64.sme.readz.q.vert.nxv4f32(i32, i32)903declare <vscale x 2 x double> @llvm.aarch64.sme.readz.q.vert.nxv2f64(i32, i32)904 905;MOVAZ (array to vector, Multi)906 907 908;;909; X2910;;911 912define {<vscale x 16 x i8>, <vscale x 16 x i8>} @test_readz_z8_i8_x2(i32 %slice) #0 {913; CHECK-LABEL: test_readz_z8_i8_x2:914; CHECK: // %bb.0:915; CHECK-NEXT: mov w8, w0916; CHECK-NEXT: movaz { z0.d, z1.d }, za.d[w8, 0, vgx2]917; CHECK-NEXT: movaz { z0.d, z1.d }, za.d[w8, 7, vgx2]918; CHECK-NEXT: ret919 %res = call {<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sme.readz.x2.nxv16i8(i32 %slice)920 %slice.max = add i32 %slice, 7921 %res2 = call {<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sme.readz.x2.nxv16i8(i32 %slice.max)922 ret {<vscale x 16 x i8>, <vscale x 16 x i8>} %res2923}924 925define {<vscale x 8 x i16>, <vscale x 8 x i16>} @test_readz_z16_i16_x2(i32 %slice) #0 {926; CHECK-LABEL: test_readz_z16_i16_x2:927; CHECK: // %bb.0:928; CHECK-NEXT: mov w8, w0929; CHECK-NEXT: movaz { z0.d, z1.d }, za.d[w8, 0, vgx2]930; CHECK-NEXT: movaz { z0.d, z1.d }, za.d[w8, 7, vgx2]931; CHECK-NEXT: ret932 %res = call {<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.aarch64.sme.readz.x2.nxv8i16(i32 %slice)933 %slice.max = add i32 %slice, 7934 %res2 = call {<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.aarch64.sme.readz.x2.nxv8i16(i32 %slice.max)935 ret {<vscale x 8 x i16>, <vscale x 8 x i16>} %res2936}937 938define {<vscale x 4 x i32>, <vscale x 4 x i32>} @test_readz_z32_i32_x2(i32 %slice) #0 {939; CHECK-LABEL: test_readz_z32_i32_x2:940; CHECK: // %bb.0:941; CHECK-NEXT: mov w8, w0942; CHECK-NEXT: movaz { z0.d, z1.d }, za.d[w8, 0, vgx2]943; CHECK-NEXT: movaz { z0.d, z1.d }, za.d[w8, 7, vgx2]944; CHECK-NEXT: ret945 %res = call {<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.aarch64.sme.readz.x2.nxv4i32(i32 %slice)946 %slice.max = add i32 %slice, 7947 %res2 = call {<vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.aarch64.sme.readz.x2.nxv4i32(i32 %slice.max)948 ret {<vscale x 4 x i32>, <vscale x 4 x i32>} %res2949}950 951define {<vscale x 2 x i64>, <vscale x 2 x i64>} @test_readz_z64_i64_x2(i32 %slice) #0 {952; CHECK-LABEL: test_readz_z64_i64_x2:953; CHECK: // %bb.0:954; CHECK-NEXT: mov w8, w0955; CHECK-NEXT: movaz { z0.d, z1.d }, za.d[w8, 0, vgx2]956; CHECK-NEXT: movaz { z0.d, z1.d }, za.d[w8, 7, vgx2]957; CHECK-NEXT: ret958 %res = call {<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.aarch64.sme.readz.x2.nxv2i64(i32 %slice)959 %slice.max = add i32 %slice, 7960 %res2 = call {<vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.aarch64.sme.readz.x2.nxv2i64(i32 %slice.max)961 ret {<vscale x 2 x i64>, <vscale x 2 x i64>} %res2962}963 964define {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @test_readz_z16_bf16_x2(i32 %slice) #0 {965; CHECK-LABEL: test_readz_z16_bf16_x2:966; CHECK: // %bb.0:967; CHECK-NEXT: mov w8, w0968; CHECK-NEXT: movaz { z0.d, z1.d }, za.d[w8, 0, vgx2]969; CHECK-NEXT: movaz { z0.d, z1.d }, za.d[w8, 7, vgx2]970; CHECK-NEXT: ret971 %res = call {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.aarch64.sme.readz.x2.nxv8bf16(i32 %slice)972 %slice.max = add i32 %slice, 7973 %res2 = call {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.aarch64.sme.readz.x2.nxv8bf16(i32 %slice.max)974 ret {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>} %res2975}976 977define {<vscale x 8 x half>, <vscale x 8 x half>} @test_readz_z16_f16_x2(i32 %slice) #0 {978; CHECK-LABEL: test_readz_z16_f16_x2:979; CHECK: // %bb.0:980; CHECK-NEXT: mov w8, w0981; CHECK-NEXT: movaz { z0.d, z1.d }, za.d[w8, 0, vgx2]982; CHECK-NEXT: movaz { z0.d, z1.d }, za.d[w8, 7, vgx2]983; CHECK-NEXT: ret984 %res = call {<vscale x 8 x half>, <vscale x 8 x half>} @llvm.aarch64.sme.readz.x2.nxv8f16(i32 %slice)985 %slice.max = add i32 %slice, 7986 %res2 = call {<vscale x 8 x half>, <vscale x 8 x half>} @llvm.aarch64.sme.readz.x2.nxv8f16(i32 %slice.max)987 ret {<vscale x 8 x half>, <vscale x 8 x half>} %res2988}989 990define {<vscale x 4 x float>, <vscale x 4 x float>} @test_readz_z32_f32_x2(i32 %slice) #0 {991; CHECK-LABEL: test_readz_z32_f32_x2:992; CHECK: // %bb.0:993; CHECK-NEXT: mov w8, w0994; CHECK-NEXT: movaz { z0.d, z1.d }, za.d[w8, 0, vgx2]995; CHECK-NEXT: movaz { z0.d, z1.d }, za.d[w8, 7, vgx2]996; CHECK-NEXT: ret997 %res = call {<vscale x 4 x float>, <vscale x 4 x float>} @llvm.aarch64.sme.readz.x2.nxv4f32(i32 %slice)998 %slice.max = add i32 %slice, 7999 %res2 = call {<vscale x 4 x float>, <vscale x 4 x float>} @llvm.aarch64.sme.readz.x2.nxv4f32(i32 %slice.max)1000 ret {<vscale x 4 x float>, <vscale x 4 x float>} %res21001}1002 1003define {<vscale x 2 x double>, <vscale x 2 x double>} @test_readz_z64_f64_x2(i32 %slice) #0 {1004; CHECK-LABEL: test_readz_z64_f64_x2:1005; CHECK: // %bb.0:1006; CHECK-NEXT: mov w8, w01007; CHECK-NEXT: movaz { z0.d, z1.d }, za.d[w8, 0, vgx2]1008; CHECK-NEXT: movaz { z0.d, z1.d }, za.d[w8, 7, vgx2]1009; CHECK-NEXT: ret1010 %res = call {<vscale x 2 x double>, <vscale x 2 x double>} @llvm.aarch64.sme.readz.x2.nxv2f64(i32 %slice)1011 %slice.max = add i32 %slice, 71012 %res2 = call {<vscale x 2 x double>, <vscale x 2 x double>} @llvm.aarch64.sme.readz.x2.nxv2f64(i32 %slice.max)1013 ret {<vscale x 2 x double>, <vscale x 2 x double>} %res21014}1015 1016;;1017; X41018;;1019 1020define {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @test_readz_z8_i8_x4(i32 %slice) #0 {1021; CHECK-LABEL: test_readz_z8_i8_x4:1022; CHECK: // %bb.0:1023; CHECK-NEXT: mov w8, w01024; CHECK-NEXT: movaz { z0.d - z3.d }, za.d[w8, 0, vgx4]1025; CHECK-NEXT: movaz { z0.d - z3.d }, za.d[w8, 7, vgx4]1026; CHECK-NEXT: ret1027 %res = call {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sme.readz.x4.nxv16i8(i32 %slice)1028 %slice.max = add i32 %slice, 71029 %res2 = call {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sme.readz.x4.nxv16i8(i32 %slice.max)1030 ret {<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} %res21031}1032 1033define {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} @test_readz_z16_i16_x4(i32 %slice) #0 {1034; CHECK-LABEL: test_readz_z16_i16_x4:1035; CHECK: // %bb.0:1036; CHECK-NEXT: mov w8, w01037; CHECK-NEXT: movaz { z0.d - z3.d }, za.d[w8, 0, vgx4]1038; CHECK-NEXT: movaz { z0.d - z3.d }, za.d[w8, 7, vgx4]1039; CHECK-NEXT: ret1040 %res = call {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.aarch64.sme.readz.x4.nxv8i16(i32 %slice)1041 %slice.max = add i32 %slice, 71042 %res2 = call {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.aarch64.sme.readz.x4.nxv8i16(i32 %slice.max)1043 ret {<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>} %res21044}1045 1046define {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} @test_readz_z32_i32_x4(i32 %slice) #0 {1047; CHECK-LABEL: test_readz_z32_i32_x4:1048; CHECK: // %bb.0:1049; CHECK-NEXT: mov w8, w01050; CHECK-NEXT: movaz { z0.d - z3.d }, za.d[w8, 0, vgx4]1051; CHECK-NEXT: movaz { z0.d - z3.d }, za.d[w8, 7, vgx4]1052; CHECK-NEXT: ret1053 %res = call {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.aarch64.sme.readz.x4.nxv4i32(i32 %slice)1054 %slice.max = add i32 %slice, 71055 %res2 = call {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} @llvm.aarch64.sme.readz.x4.nxv4i32(i32 %slice.max)1056 ret {<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>} %res21057}1058 1059define {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @test_readz_z64_i64_x4(i32 %slice) #0 {1060; CHECK-LABEL: test_readz_z64_i64_x4:1061; CHECK: // %bb.0:1062; CHECK-NEXT: mov w8, w01063; CHECK-NEXT: movaz { z0.d - z3.d }, za.d[w8, 0, vgx4]1064; CHECK-NEXT: movaz { z0.d - z3.d }, za.d[w8, 7, vgx4]1065; CHECK-NEXT: ret1066 %res = call {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.aarch64.sme.readz.x4.nxv2i64(i32 %slice)1067 %slice.max = add i32 %slice, 71068 %res2 = call {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} @llvm.aarch64.sme.readz.x4.nxv2i64(i32 %slice.max)1069 ret {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>} %res21070}1071 1072define {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @test_readz_z16_bf16_x4(i32 %slice) #0 {1073; CHECK-LABEL: test_readz_z16_bf16_x4:1074; CHECK: // %bb.0:1075; CHECK-NEXT: mov w8, w01076; CHECK-NEXT: movaz { z0.d - z3.d }, za.d[w8, 0, vgx4]1077; CHECK-NEXT: movaz { z0.d - z3.d }, za.d[w8, 7, vgx4]1078; CHECK-NEXT: ret1079 %res = call {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.aarch64.sme.readz.x4.nxv8bf16(i32 %slice)1080 %slice.max = add i32 %slice, 71081 %res2 = call {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>} @llvm.aarch64.sme.readz.x4.nxv8bf16(i32 %slice.max)1082 ret {<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>} %res21083}1084 1085define {<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>} @test_readz_z16_f16_x4(i32 %slice) #0 {1086; CHECK-LABEL: test_readz_z16_f16_x4:1087; CHECK: // %bb.0:1088; CHECK-NEXT: mov w8, w01089; CHECK-NEXT: movaz { z0.d - z3.d }, za.d[w8, 0, vgx4]1090; CHECK-NEXT: movaz { z0.d - z3.d }, za.d[w8, 7, vgx4]1091; CHECK-NEXT: ret1092 %res = call {<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>} @llvm.aarch64.sme.readz.x4.nxv8f16(i32 %slice)1093 %slice.max = add i32 %slice, 71094 %res2 = call {<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>} @llvm.aarch64.sme.readz.x4.nxv8f16(i32 %slice.max)1095 ret {<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>} %res21096}1097 1098define {<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>} @test_readz_z32_f32_x4(i32 %slice) #0 {1099; CHECK-LABEL: test_readz_z32_f32_x4:1100; CHECK: // %bb.0:1101; CHECK-NEXT: mov w8, w01102; CHECK-NEXT: movaz { z0.d - z3.d }, za.d[w8, 0, vgx4]1103; CHECK-NEXT: movaz { z0.d - z3.d }, za.d[w8, 7, vgx4]1104; CHECK-NEXT: ret1105 %res = call {<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>} @llvm.aarch64.sme.readz.x4.nxv4f32(i32 %slice)1106 %slice.max = add i32 %slice, 71107 %res2 = call {<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>} @llvm.aarch64.sme.readz.x4.nxv4f32(i32 %slice.max)1108 ret {<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>} %res21109}1110 1111define {<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>} @test_readz_z64_f64_x4(i32 %slice) #0 {1112; CHECK-LABEL: test_readz_z64_f64_x4:1113; CHECK: // %bb.0:1114; CHECK-NEXT: mov w8, w01115; CHECK-NEXT: movaz { z0.d - z3.d }, za.d[w8, 0, vgx4]1116; CHECK-NEXT: movaz { z0.d - z3.d }, za.d[w8, 7, vgx4]1117; CHECK-NEXT: ret1118 %res = call {<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>} @llvm.aarch64.sme.readz.x4.nxv2f64(i32 %slice)1119 %slice.max = add i32 %slice, 71120 %res2 = call {<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>} @llvm.aarch64.sme.readz.x4.nxv2f64(i32 %slice.max)1121 ret {<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>} %res21122}1123 1124attributes #0 = { "target-features"="+sme2p1" }1125