530 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming -verify-machineinstrs < %s | FileCheck %s3 4define void @extract_row_b(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 %tileslice) {5; CHECK-LABEL: extract_row_b:6; CHECK: // %bb.0:7; CHECK-NEXT: mov z7.d, z0.d8; CHECK-NEXT: mov z1.d, z0.d9; CHECK-NEXT: mov w12, w010; CHECK-NEXT: mov z2.d, z0.d11; CHECK-NEXT: mov z3.d, z0.d12; CHECK-NEXT: mov z0.b, p0/m, za0h.b[w12, 0]13; CHECK-NEXT: mov z4.d, z7.d14; CHECK-NEXT: mov z5.d, z7.d15; CHECK-NEXT: mov z6.d, z7.d16; CHECK-NEXT: mov z1.b, p0/m, za0h.b[w12, 2]17; CHECK-NEXT: mov z2.b, p0/m, za0h.b[w12, 4]18; CHECK-NEXT: mov z3.b, p0/m, za0h.b[w12, 6]19; CHECK-NEXT: mov z4.b, p0/m, za0h.b[w12, 8]20; CHECK-NEXT: mov z5.b, p0/m, za0h.b[w12, 10]21; CHECK-NEXT: mov z6.b, p0/m, za0h.b[w12, 12]22; CHECK-NEXT: mov z7.b, p0/m, za0h.b[w12, 14]23; CHECK-NEXT: b use24 %z0 = call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 0, i32 %tileslice)25 %tileslice.2 = add i32 %tileslice, 226 %z1 = call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 0, i32 %tileslice.2)27 %tileslice.4 = add i32 %tileslice, 428 %z2 = call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 0, i32 %tileslice.4)29 %tileslice.6 = add i32 %tileslice, 630 %z3 = call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 0, i32 %tileslice.6)31 %tileslice.8 = add i32 %tileslice, 832 %z4 = call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 0, i32 %tileslice.8)33 %tileslice.10 = add i32 %tileslice, 1034 %z5 = call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 0, i32 %tileslice.10)35 %tileslice.12 = add i32 %tileslice, 1236 %z6 = call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 0, i32 %tileslice.12)37 %tileslice.14 = add i32 %tileslice, 1438 %z7 = call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 0, i32 %tileslice.14)39 40 ; Force retention of z0..z741 tail call void @use(<vscale x 16 x i8> %z0, <vscale x 16 x i8> %z1, <vscale x 16 x i8> %z2, <vscale x 16 x i8> %z3,42 <vscale x 16 x i8> %z4, <vscale x 16 x i8> %z5, <vscale x 16 x i8> %z6, <vscale x 16 x i8> %z7)43 ret void44}45 46define void @extract_col_b(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 %tileslice) {47; CHECK-LABEL: extract_col_b:48; CHECK: // %bb.0:49; CHECK-NEXT: mov z7.d, z0.d50; CHECK-NEXT: mov z1.d, z0.d51; CHECK-NEXT: mov w12, w052; CHECK-NEXT: mov z2.d, z0.d53; CHECK-NEXT: mov z3.d, z0.d54; CHECK-NEXT: mov z0.b, p0/m, za0v.b[w12, 1]55; CHECK-NEXT: mov z4.d, z7.d56; CHECK-NEXT: mov z5.d, z7.d57; CHECK-NEXT: mov z6.d, z7.d58; CHECK-NEXT: mov z1.b, p0/m, za0v.b[w12, 3]59; CHECK-NEXT: mov z2.b, p0/m, za0v.b[w12, 5]60; CHECK-NEXT: mov z3.b, p0/m, za0v.b[w12, 7]61; CHECK-NEXT: mov z4.b, p0/m, za0v.b[w12, 9]62; CHECK-NEXT: mov z5.b, p0/m, za0v.b[w12, 11]63; CHECK-NEXT: mov z6.b, p0/m, za0v.b[w12, 13]64; CHECK-NEXT: mov z7.b, p0/m, za0v.b[w12, 15]65; CHECK-NEXT: b use66 %tileslice.1 = add i32 %tileslice, 167 %z0 = call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 0, i32 %tileslice.1)68 %tileslice.3 = add i32 %tileslice, 369 %z1 = call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 0, i32 %tileslice.3)70 %tileslice.5 = add i32 %tileslice, 571 %z2 = call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 0, i32 %tileslice.5)72 %tileslice.7 = add i32 %tileslice, 773 %z3 = call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 0, i32 %tileslice.7)74 %tileslice.9 = add i32 %tileslice, 975 %z4 = call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 0, i32 %tileslice.9)76 %tileslice.11 = add i32 %tileslice, 1177 %z5 = call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 0, i32 %tileslice.11)78 %tileslice.13 = add i32 %tileslice, 1379 %z6 = call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 0, i32 %tileslice.13)80 %tileslice.15 = add i32 %tileslice, 1581 %z7 = call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 0, i32 %tileslice.15)82 83 tail call void @use(<vscale x 16 x i8> %z0, <vscale x 16 x i8> %z1, <vscale x 16 x i8> %z2, <vscale x 16 x i8> %z3,84 <vscale x 16 x i8> %z4, <vscale x 16 x i8> %z5, <vscale x 16 x i8> %z6, <vscale x 16 x i8> %z7)85 ret void86}87 88define void @extract_row_h(<vscale x 8 x i16> %zd, <vscale x 8 x i1> %pg, i32 %tileslice) {89; CHECK-LABEL: extract_row_h:90; CHECK: // %bb.0:91; CHECK-NEXT: mov z3.d, z0.d92; CHECK-NEXT: mov z1.d, z0.d93; CHECK-NEXT: mov w12, w094; CHECK-NEXT: mov z2.d, z0.d95; CHECK-NEXT: mov z0.h, p0/m, za0h.h[w12, 0]96; CHECK-NEXT: mov z1.h, p0/m, za0h.h[w12, 2]97; CHECK-NEXT: mov z2.h, p0/m, za0h.h[w12, 4]98; CHECK-NEXT: mov z3.h, p0/m, za0h.h[w12, 6]99; CHECK-NEXT: b use100 %z0 = call <vscale x 8 x i16> @llvm.aarch64.sme.read.horiz.nxv8i16(<vscale x 8 x i16> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice)101 %tileslice.2 = add i32 %tileslice, 2102 %z1 = call <vscale x 8 x i16> @llvm.aarch64.sme.read.horiz.nxv8i16(<vscale x 8 x i16> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice.2)103 %tileslice.4 = add i32 %tileslice, 4104 %z2 = call <vscale x 8 x i16> @llvm.aarch64.sme.read.horiz.nxv8i16(<vscale x 8 x i16> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice.4)105 %tileslice.6 = add i32 %tileslice, 6106 %z3 = call <vscale x 8 x i16> @llvm.aarch64.sme.read.horiz.nxv8i16(<vscale x 8 x i16> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice.6)107 108 tail call void @use(<vscale x 8 x i16> %z0, <vscale x 8 x i16> %z1, <vscale x 8 x i16> %z2, <vscale x 8 x i16> %z3)109 ret void110}111 112define void @extract_col_h(<vscale x 8 x i16> %zd, <vscale x 8 x i1> %pg, i32 %tileslice) {113; CHECK-LABEL: extract_col_h:114; CHECK: // %bb.0:115; CHECK-NEXT: mov z3.d, z0.d116; CHECK-NEXT: mov z1.d, z0.d117; CHECK-NEXT: mov w12, w0118; CHECK-NEXT: mov z2.d, z0.d119; CHECK-NEXT: mov z0.h, p0/m, za1v.h[w12, 1]120; CHECK-NEXT: mov z1.h, p0/m, za1v.h[w12, 3]121; CHECK-NEXT: mov z2.h, p0/m, za1v.h[w12, 5]122; CHECK-NEXT: mov z3.h, p0/m, za1v.h[w12, 7]123; CHECK-NEXT: b use124 %tileslice.1 = add i32 %tileslice, 1125 %z0 = call <vscale x 8 x i16> @llvm.aarch64.sme.read.vert.nxv8i16(<vscale x 8 x i16> %zd, <vscale x 8 x i1> %pg, i32 1, i32 %tileslice.1)126 %tileslice.3 = add i32 %tileslice, 3127 %z1 = call <vscale x 8 x i16> @llvm.aarch64.sme.read.vert.nxv8i16(<vscale x 8 x i16> %zd, <vscale x 8 x i1> %pg, i32 1, i32 %tileslice.3)128 %tileslice.5 = add i32 %tileslice, 5129 %z2 = call <vscale x 8 x i16> @llvm.aarch64.sme.read.vert.nxv8i16(<vscale x 8 x i16> %zd, <vscale x 8 x i1> %pg, i32 1, i32 %tileslice.5)130 %tileslice.7 = add i32 %tileslice, 7131 %z3 = call <vscale x 8 x i16> @llvm.aarch64.sme.read.vert.nxv8i16(<vscale x 8 x i16> %zd, <vscale x 8 x i1> %pg, i32 1, i32 %tileslice.7)132 133 tail call void @use(<vscale x 8 x i16> %z0, <vscale x 8 x i16> %z1, <vscale x 8 x i16> %z2, <vscale x 8 x i16> %z3)134 ret void135}136 137define void @extract_f16(<vscale x 8 x half> %zd, <vscale x 8 x i1> %pg, i32 %tileslice) {138; CHECK-LABEL: extract_f16:139; CHECK: // %bb.0:140; CHECK-NEXT: mov z7.d, z0.d141; CHECK-NEXT: mov z1.d, z0.d142; CHECK-NEXT: mov w12, w0143; CHECK-NEXT: mov z2.d, z0.d144; CHECK-NEXT: mov z3.d, z0.d145; CHECK-NEXT: mov z0.h, p0/m, za0h.h[w12, 0]146; CHECK-NEXT: mov z4.d, z7.d147; CHECK-NEXT: mov z5.d, z7.d148; CHECK-NEXT: mov z6.d, z7.d149; CHECK-NEXT: mov z1.h, p0/m, za0h.h[w12, 1]150; CHECK-NEXT: mov z2.h, p0/m, za0v.h[w12, 2]151; CHECK-NEXT: mov z3.h, p0/m, za0v.h[w12, 3]152; CHECK-NEXT: mov z4.h, p0/m, za0h.h[w12, 4]153; CHECK-NEXT: mov z5.h, p0/m, za0h.h[w12, 5]154; CHECK-NEXT: mov z6.h, p0/m, za0v.h[w12, 6]155; CHECK-NEXT: mov z7.h, p0/m, za0v.h[w12, 7]156; CHECK-NEXT: b use157 %z0 = call <vscale x 8 x half> @llvm.aarch64.sme.read.horiz.nxv8f16(<vscale x 8 x half> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice)158 %tileslice.1 = add i32 %tileslice, 1159 %z1 = call <vscale x 8 x half> @llvm.aarch64.sme.read.horiz.nxv8f16(<vscale x 8 x half> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice.1)160 %tileslice.2 = add i32 %tileslice, 2161 %z2 = call <vscale x 8 x half> @llvm.aarch64.sme.read.vert.nxv8f16(<vscale x 8 x half> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice.2)162 %tileslice.3 = add i32 %tileslice, 3163 %z3 = call <vscale x 8 x half> @llvm.aarch64.sme.read.vert.nxv8f16(<vscale x 8 x half> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice.3)164 %tileslice.4 = add i32 %tileslice, 4165 %z4 = call <vscale x 8 x half> @llvm.aarch64.sme.read.horiz.nxv8f16(<vscale x 8 x half> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice.4)166 %tileslice.5 = add i32 %tileslice, 5167 %z5 = call <vscale x 8 x half> @llvm.aarch64.sme.read.horiz.nxv8f16(<vscale x 8 x half> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice.5)168 %tileslice.6 = add i32 %tileslice, 6169 %z6 = call <vscale x 8 x half> @llvm.aarch64.sme.read.vert.nxv8f16(<vscale x 8 x half> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice.6)170 %tileslice.7 = add i32 %tileslice, 7171 %z7 = call <vscale x 8 x half> @llvm.aarch64.sme.read.vert.nxv8f16(<vscale x 8 x half> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice.7)172 173 tail call void @use(<vscale x 8 x half> %z0, <vscale x 8 x half> %z1, <vscale x 8 x half> %z2, <vscale x 8 x half> %z3,174 <vscale x 8 x half> %z4, <vscale x 8 x half> %z5, <vscale x 8 x half> %z6, <vscale x 8 x half> %z7)175 ret void176}177 178define void @extract_bf16(<vscale x 8 x bfloat> %zd, <vscale x 8 x i1> %pg, i32 %tileslice, ptr %ptr) {179; CHECK-LABEL: extract_bf16:180; CHECK: // %bb.0:181; CHECK-NEXT: mov z7.d, z0.d182; CHECK-NEXT: mov z1.d, z0.d183; CHECK-NEXT: mov w12, w0184; CHECK-NEXT: mov z2.d, z0.d185; CHECK-NEXT: mov z3.d, z0.d186; CHECK-NEXT: mov z0.h, p0/m, za0h.h[w12, 0]187; CHECK-NEXT: mov z4.d, z7.d188; CHECK-NEXT: mov z5.d, z7.d189; CHECK-NEXT: mov z6.d, z7.d190; CHECK-NEXT: mov z1.h, p0/m, za0h.h[w12, 1]191; CHECK-NEXT: mov z2.h, p0/m, za0v.h[w12, 2]192; CHECK-NEXT: mov z3.h, p0/m, za0v.h[w12, 3]193; CHECK-NEXT: mov z4.h, p0/m, za0h.h[w12, 4]194; CHECK-NEXT: mov z5.h, p0/m, za0h.h[w12, 5]195; CHECK-NEXT: mov z6.h, p0/m, za0v.h[w12, 6]196; CHECK-NEXT: mov z7.h, p0/m, za0v.h[w12, 7]197; CHECK-NEXT: b use198 %z0 = call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.horiz.nxv8bf16(<vscale x 8 x bfloat> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice)199 %tileslice.1 = add i32 %tileslice, 1200 %z1 = call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.horiz.nxv8bf16(<vscale x 8 x bfloat> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice.1)201 %tileslice.2 = add i32 %tileslice, 2202 %z2 = call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.vert.nxv8bf16(<vscale x 8 x bfloat> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice.2)203 %tileslice.3 = add i32 %tileslice, 3204 %z3 = call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.vert.nxv8bf16(<vscale x 8 x bfloat> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice.3)205 %tileslice.4 = add i32 %tileslice, 4206 %z4 = call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.horiz.nxv8bf16(<vscale x 8 x bfloat> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice.4)207 %tileslice.5 = add i32 %tileslice, 5208 %z5 = call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.horiz.nxv8bf16(<vscale x 8 x bfloat> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice.5)209 %tileslice.6 = add i32 %tileslice, 6210 %z6 = call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.vert.nxv8bf16(<vscale x 8 x bfloat> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice.6)211 %tileslice.7 = add i32 %tileslice, 7212 %z7 = call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.vert.nxv8bf16(<vscale x 8 x bfloat> %zd, <vscale x 8 x i1> %pg, i32 0, i32 %tileslice.7)213 214 tail call void @use(<vscale x 8 x bfloat> %z0, <vscale x 8 x bfloat> %z1, <vscale x 8 x bfloat> %z2, <vscale x 8 x bfloat> %z3,215 <vscale x 8 x bfloat> %z4, <vscale x 8 x bfloat> %z5, <vscale x 8 x bfloat> %z6, <vscale x 8 x bfloat> %z7)216 ret void217}218 219define void @extract_row_s(<vscale x 4 x i32> %zd, <vscale x 4 x i1> %pg, i32 %tileslice) {220; CHECK-LABEL: extract_row_s:221; CHECK: // %bb.0:222; CHECK-NEXT: mov z1.d, z0.d223; CHECK-NEXT: mov w12, w0224; CHECK-NEXT: mov z0.s, p0/m, za0h.s[w12, 0]225; CHECK-NEXT: mov z1.s, p0/m, za0h.s[w12, 2]226; CHECK-NEXT: b use227 %z0 = call <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32> %zd, <vscale x 4 x i1> %pg, i32 0, i32 %tileslice)228 %tileslice.2 = add i32 %tileslice, 2229 %z1 = call <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32> %zd, <vscale x 4 x i1> %pg, i32 0, i32 %tileslice.2)230 231 tail call void @use(<vscale x 4 x i32> %z0, <vscale x 4 x i32> %z1)232 ret void233}234 235define void @extract_col_s(<vscale x 4 x i32> %zd, <vscale x 4 x i1> %pg, i32 %tileslice) {236; CHECK-LABEL: extract_col_s:237; CHECK: // %bb.0:238; CHECK-NEXT: mov z1.d, z0.d239; CHECK-NEXT: mov w12, w0240; CHECK-NEXT: mov z0.s, p0/m, za3v.s[w12, 1]241; CHECK-NEXT: mov z1.s, p0/m, za3v.s[w12, 3]242; CHECK-NEXT: b use243 %tileslice.1 = add i32 %tileslice, 1244 %z0 = call <vscale x 4 x i32> @llvm.aarch64.sme.read.vert.nxv4i32(<vscale x 4 x i32> %zd, <vscale x 4 x i1> %pg, i32 3, i32 %tileslice.1)245 %tileslice.3 = add i32 %tileslice, 3246 %z1 = call <vscale x 4 x i32> @llvm.aarch64.sme.read.vert.nxv4i32(<vscale x 4 x i32> %zd, <vscale x 4 x i1> %pg, i32 3, i32 %tileslice.3)247 248 tail call void @use(<vscale x 4 x i32> %z0, <vscale x 4 x i32> %z1)249 ret void250}251 252define void @extract_f32(<vscale x 4 x float> %zd, <vscale x 4 x i1> %pg, i32 %tileslice) {253; CHECK-LABEL: extract_f32:254; CHECK: // %bb.0:255; CHECK-NEXT: mov z3.d, z0.d256; CHECK-NEXT: mov z1.d, z0.d257; CHECK-NEXT: mov w12, w0258; CHECK-NEXT: mov z2.d, z0.d259; CHECK-NEXT: mov z0.s, p0/m, za0h.s[w12, 0]260; CHECK-NEXT: mov z1.s, p0/m, za0h.s[w12, 1]261; CHECK-NEXT: mov z2.s, p0/m, za0v.s[w12, 2]262; CHECK-NEXT: mov z3.s, p0/m, za0v.s[w12, 3]263; CHECK-NEXT: b use264 %z0 = call <vscale x 4 x float> @llvm.aarch64.sme.read.horiz.nxv4f32(<vscale x 4 x float> %zd, <vscale x 4 x i1> %pg, i32 0, i32 %tileslice)265 %tileslice.1 = add i32 %tileslice, 1266 %z1 = call <vscale x 4 x float> @llvm.aarch64.sme.read.horiz.nxv4f32(<vscale x 4 x float> %zd, <vscale x 4 x i1> %pg, i32 0, i32 %tileslice.1)267 %tileslice.2 = add i32 %tileslice, 2268 %z2 = call <vscale x 4 x float> @llvm.aarch64.sme.read.vert.nxv4f32(<vscale x 4 x float> %zd, <vscale x 4 x i1> %pg, i32 0, i32 %tileslice.2)269 %tileslice.3 = add i32 %tileslice, 3270 %z3 = call <vscale x 4 x float> @llvm.aarch64.sme.read.vert.nxv4f32(<vscale x 4 x float> %zd, <vscale x 4 x i1> %pg, i32 0, i32 %tileslice.3)271 272 tail call void @use(<vscale x 4 x float> %z0, <vscale x 4 x float> %z1, <vscale x 4 x float> %z2, <vscale x 4 x float> %z3)273 ret void274}275 276define <vscale x 2 x i64> @extract_row_d(<vscale x 2 x i64> %zd, <vscale x 2 x i1> %pg, i32 %tileslice) {277; CHECK-LABEL: extract_row_d:278; CHECK: // %bb.0:279; CHECK-NEXT: mov w12, w0280; CHECK-NEXT: mov z0.d, p0/m, za0h.d[w12, 0]281; CHECK-NEXT: ret282 %z0 = call <vscale x 2 x i64> @llvm.aarch64.sme.read.horiz.nxv2i64(<vscale x 2 x i64> %zd, <vscale x 2 x i1> %pg, i32 0, i32 %tileslice)283 ret <vscale x 2 x i64> %z0284}285 286define <vscale x 2 x i64> @extract_col_d(<vscale x 2 x i64> %zd, <vscale x 2 x i1> %pg, i32 %tileslice) {287; CHECK-LABEL: extract_col_d:288; CHECK: // %bb.0:289; CHECK-NEXT: mov w12, w0290; CHECK-NEXT: mov z0.d, p0/m, za1v.d[w12, 1]291; CHECK-NEXT: ret292 %tileslice.1 = add i32 %tileslice, 1293 %z0 = call <vscale x 2 x i64> @llvm.aarch64.sme.read.vert.nxv2i64(<vscale x 2 x i64> %zd, <vscale x 2 x i1> %pg, i32 1, i32 %tileslice.1)294 ret <vscale x 2 x i64> %z0295}296 297define void @extract_f64(<vscale x 2 x double> %zd, <vscale x 2 x i1> %pg, i32 %tileslice) {298; CHECK-LABEL: extract_f64:299; CHECK: // %bb.0:300; CHECK-NEXT: mov z1.d, z0.d301; CHECK-NEXT: mov w12, w0302; CHECK-NEXT: mov z0.d, p0/m, za0h.d[w12, 0]303; CHECK-NEXT: mov z1.d, p0/m, za0v.d[w12, 1]304; CHECK-NEXT: b use305 %z0 = call <vscale x 2 x double> @llvm.aarch64.sme.read.horiz.nxv2f64(<vscale x 2 x double> %zd, <vscale x 2 x i1> %pg, i32 0, i32 %tileslice)306 %tileslice.1 = add i32 %tileslice, 1307 %z1 = call <vscale x 2 x double> @llvm.aarch64.sme.read.vert.nxv2f64(<vscale x 2 x double> %zd, <vscale x 2 x i1> %pg, i32 0, i32 %tileslice.1)308 309 tail call void @use(<vscale x 2 x double> %z0, <vscale x 2 x double> %z1)310 ret void311}312 313define <vscale x 16 x i8> @extract_row_q_v16i18(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg) {314; CHECK-LABEL: extract_row_q_v16i18:315; CHECK: // %bb.0:316; CHECK-NEXT: mov w12, wzr317; CHECK-NEXT: mov z0.q, p0/m, za0h.q[w12, 0]318; CHECK-NEXT: ret319 %res = call <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 0, i32 0)320 ret <vscale x 16 x i8> %res321}322 323define <vscale x 8 x i16> @extract_row_q_v8i16(<vscale x 8 x i16> %zd, <vscale x 8 x i1> %pg) {324; CHECK-LABEL: extract_row_q_v8i16:325; CHECK: // %bb.0:326; CHECK-NEXT: mov w12, wzr327; CHECK-NEXT: mov z0.q, p0/m, za0h.q[w12, 0]328; CHECK-NEXT: ret329 %res = call <vscale x 8 x i16> @llvm.aarch64.sme.readq.horiz.nxv8i16(<vscale x 8 x i16> %zd, <vscale x 8 x i1> %pg, i32 0, i32 0)330 ret <vscale x 8 x i16> %res331}332 333define <vscale x 8 x half> @extract_row_q_v8f16(<vscale x 8 x half> %zd, <vscale x 8 x i1> %pg) {334; CHECK-LABEL: extract_row_q_v8f16:335; CHECK: // %bb.0:336; CHECK-NEXT: mov w12, wzr337; CHECK-NEXT: mov z0.q, p0/m, za0h.q[w12, 0]338; CHECK-NEXT: ret339 %res = call <vscale x 8 x half> @llvm.aarch64.sme.readq.horiz.nxv8f16(<vscale x 8 x half> %zd, <vscale x 8 x i1> %pg, i32 0, i32 0)340 ret <vscale x 8 x half> %res341}342 343define <vscale x 4 x i32> @extract_row_q_v4i32(<vscale x 4 x i32> %zd, <vscale x 4 x i1> %pg) {344; CHECK-LABEL: extract_row_q_v4i32:345; CHECK: // %bb.0:346; CHECK-NEXT: mov w12, wzr347; CHECK-NEXT: mov z0.q, p0/m, za0h.q[w12, 0]348; CHECK-NEXT: ret349 %res = call <vscale x 4 x i32> @llvm.aarch64.sme.readq.horiz.nxv4i32(<vscale x 4 x i32> %zd, <vscale x 4 x i1> %pg, i32 0, i32 0)350 ret <vscale x 4 x i32> %res351}352 353define <vscale x 4 x float> @extract_row_q_v4f32(<vscale x 4 x float> %zd, <vscale x 4 x i1> %pg) {354; CHECK-LABEL: extract_row_q_v4f32:355; CHECK: // %bb.0:356; CHECK-NEXT: mov w12, wzr357; CHECK-NEXT: mov z0.q, p0/m, za0h.q[w12, 0]358; CHECK-NEXT: ret359 %res = call <vscale x 4 x float> @llvm.aarch64.sme.readq.horiz.nxv4f32(<vscale x 4 x float> %zd, <vscale x 4 x i1> %pg, i32 0, i32 0)360 ret <vscale x 4 x float> %res361}362 363define <vscale x 2 x i64> @extract_row_q_v2i64(<vscale x 2 x i64> %zd, <vscale x 2 x i1> %pg) {364; CHECK-LABEL: extract_row_q_v2i64:365; CHECK: // %bb.0:366; CHECK-NEXT: mov w12, wzr367; CHECK-NEXT: mov z0.q, p0/m, za0h.q[w12, 0]368; CHECK-NEXT: ret369 %res = call <vscale x 2 x i64> @llvm.aarch64.sme.readq.horiz.nxv2i64(<vscale x 2 x i64> %zd, <vscale x 2 x i1> %pg, i32 0, i32 0)370 ret <vscale x 2 x i64> %res371}372 373define <vscale x 2 x double> @extract_row_q_v2f64(<vscale x 2 x double> %zd, <vscale x 2 x i1> %pg) {374; CHECK-LABEL: extract_row_q_v2f64:375; CHECK: // %bb.0:376; CHECK-NEXT: mov w12, wzr377; CHECK-NEXT: mov z0.q, p0/m, za0h.q[w12, 0]378; CHECK-NEXT: ret379 %res = call <vscale x 2 x double> @llvm.aarch64.sme.readq.horiz.nxv2f64(<vscale x 2 x double> %zd, <vscale x 2 x i1> %pg, i32 0, i32 0)380 ret <vscale x 2 x double> %res381}382 383define <vscale x 16 x i8> @extract_col_q_v16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg) {384; CHECK-LABEL: extract_col_q_v16i8:385; CHECK: // %bb.0:386; CHECK-NEXT: mov w12, wzr387; CHECK-NEXT: mov z0.q, p0/m, za15v.q[w12, 0]388; CHECK-NEXT: ret389 %res = call <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8> %zd, <vscale x 16 x i1> %pg, i32 15, i32 0)390 ret <vscale x 16 x i8> %res391}392 393define <vscale x 8 x i16> @extract_col_q_v8i16(<vscale x 8 x i16> %zd, <vscale x 8 x i1> %pg) {394; CHECK-LABEL: extract_col_q_v8i16:395; CHECK: // %bb.0:396; CHECK-NEXT: mov w12, wzr397; CHECK-NEXT: mov z0.q, p0/m, za15v.q[w12, 0]398; CHECK-NEXT: ret399 %res = call <vscale x 8 x i16> @llvm.aarch64.sme.readq.vert.nxv8i16(<vscale x 8 x i16> %zd, <vscale x 8 x i1> %pg, i32 15, i32 0)400 ret <vscale x 8 x i16> %res401}402 403define <vscale x 8 x half> @extract_col_q_v8f16(<vscale x 8 x half> %zd, <vscale x 8 x i1> %pg) {404; CHECK-LABEL: extract_col_q_v8f16:405; CHECK: // %bb.0:406; CHECK-NEXT: mov w12, wzr407; CHECK-NEXT: mov z0.q, p0/m, za15v.q[w12, 0]408; CHECK-NEXT: ret409 %res = call <vscale x 8 x half> @llvm.aarch64.sme.readq.vert.nxv8f16(<vscale x 8 x half> %zd, <vscale x 8 x i1> %pg, i32 15, i32 0)410 ret <vscale x 8 x half> %res411}412 413define <vscale x 4 x i32> @extract_col_q_v4i32(<vscale x 4 x i32> %zd, <vscale x 4 x i1> %pg) {414; CHECK-LABEL: extract_col_q_v4i32:415; CHECK: // %bb.0:416; CHECK-NEXT: mov w12, wzr417; CHECK-NEXT: mov z0.q, p0/m, za15v.q[w12, 0]418; CHECK-NEXT: ret419 %res = call <vscale x 4 x i32> @llvm.aarch64.sme.readq.vert.nxv4i32(<vscale x 4 x i32> %zd, <vscale x 4 x i1> %pg, i32 15, i32 0)420 ret <vscale x 4 x i32> %res421}422 423define <vscale x 4 x float> @extract_col_q_v4f32(<vscale x 4 x float> %zd, <vscale x 4 x i1> %pg) {424; CHECK-LABEL: extract_col_q_v4f32:425; CHECK: // %bb.0:426; CHECK-NEXT: mov w12, wzr427; CHECK-NEXT: mov z0.q, p0/m, za15v.q[w12, 0]428; CHECK-NEXT: ret429 %res = call <vscale x 4 x float> @llvm.aarch64.sme.readq.vert.nxv4f32(<vscale x 4 x float> %zd, <vscale x 4 x i1> %pg, i32 15, i32 0)430 ret <vscale x 4 x float> %res431}432 433define <vscale x 2 x i64> @extract_col_q_v2i64(<vscale x 2 x i64> %zd, <vscale x 2 x i1> %pg) {434; CHECK-LABEL: extract_col_q_v2i64:435; CHECK: // %bb.0:436; CHECK-NEXT: mov w12, wzr437; CHECK-NEXT: mov z0.q, p0/m, za15v.q[w12, 0]438; CHECK-NEXT: ret439 %res = call <vscale x 2 x i64> @llvm.aarch64.sme.readq.vert.nxv2i64(<vscale x 2 x i64> %zd, <vscale x 2 x i1> %pg, i32 15, i32 0)440 ret <vscale x 2 x i64> %res441}442 443define <vscale x 2 x double> @extract_col_q_v2f64(<vscale x 2 x double> %zd, <vscale x 2 x i1> %pg) {444; CHECK-LABEL: extract_col_q_v2f64:445; CHECK: // %bb.0:446; CHECK-NEXT: mov w12, wzr447; CHECK-NEXT: mov z0.q, p0/m, za15v.q[w12, 0]448; CHECK-NEXT: ret449 %res = call <vscale x 2 x double> @llvm.aarch64.sme.readq.vert.nxv2f64(<vscale x 2 x double> %zd, <vscale x 2 x i1> %pg, i32 15, i32 0)450 ret <vscale x 2 x double> %res451}452 453define <vscale x 4 x i32> @test_sink_offset_operand(<vscale x 4 x i1> %pg, i32 %base, i32 %N) {454; CHECK-LABEL: test_sink_offset_operand:455; CHECK: // %bb.0: // %entry456; CHECK-NEXT: mov w12, w0457; CHECK-NEXT: .LBB26_1: // %for.body458; CHECK-NEXT: // =>This Inner Loop Header: Depth=1459; CHECK-NEXT: mov z0.s, #0 // =0x0460; CHECK-NEXT: mov z1.s, #0 // =0x0461; CHECK-NEXT: subs w1, w1, #3462; CHECK-NEXT: mov z2.s, #0 // =0x0463; CHECK-NEXT: mov z0.s, p0/m, za0h.s[w12, 0]464; CHECK-NEXT: mov z1.s, p0/m, za0h.s[w12, 1]465; CHECK-NEXT: mov z2.s, p0/m, za0h.s[w12, 2]466; CHECK-NEXT: b.ne .LBB26_1467; CHECK-NEXT: // %bb.2: // %exit468; CHECK-NEXT: add z0.s, z0.s, z1.s469; CHECK-NEXT: add z0.s, z0.s, z2.s470; CHECK-NEXT: ret471entry:472 %add1 = add i32 %base, 1473 %add2 = add i32 %base, 2474 br label %for.body475 476for.body:477 %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]478 %z0 = call <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, i32 0, i32 %base)479 %z1 = call <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, i32 0, i32 %add1)480 %z2 = call <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, i32 0, i32 %add2)481 %inc = add nuw nsw i32 %i, 3482 %exitcond.not = icmp eq i32 %inc, %N483 br i1 %exitcond.not, label %exit, label %for.body484 485exit:486 %tmp1 = add <vscale x 4 x i32> %z0, %z1487 %res = add <vscale x 4 x i32> %tmp1, %z2488 ret <vscale x 4 x i32> %res489}490 491declare <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i32, i32)492declare <vscale x 8 x i16> @llvm.aarch64.sme.read.horiz.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i32, i32)493declare <vscale x 8 x half> @llvm.aarch64.sme.read.horiz.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, i32, i32)494declare <vscale x 8 x bfloat> @llvm.aarch64.sme.read.horiz.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x i1>, i32, i32)495declare <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32, i32)496declare <vscale x 4 x float> @llvm.aarch64.sme.read.horiz.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, i32, i32)497declare <vscale x 2 x i64> @llvm.aarch64.sme.read.horiz.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i32, i32)498declare <vscale x 2 x double> @llvm.aarch64.sme.read.horiz.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, i32, i32)499declare <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i32, i32)500declare <vscale x 8 x i16> @llvm.aarch64.sme.read.vert.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i32, i32)501declare <vscale x 8 x half> @llvm.aarch64.sme.read.vert.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, i32, i32)502declare <vscale x 8 x bfloat> @llvm.aarch64.sme.read.vert.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x i1>, i32, i32)503declare <vscale x 4 x i32> @llvm.aarch64.sme.read.vert.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32, i32)504declare <vscale x 4 x float> @llvm.aarch64.sme.read.vert.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, i32, i32)505declare <vscale x 2 x i64> @llvm.aarch64.sme.read.vert.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i32, i32)506declare <vscale x 2 x double> @llvm.aarch64.sme.read.vert.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, i32, i32)507 508declare <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i32, i32)509declare <vscale x 8 x i16> @llvm.aarch64.sme.readq.horiz.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i32, i32)510declare <vscale x 8 x half> @llvm.aarch64.sme.readq.horiz.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, i32, i32)511declare <vscale x 8 x bfloat> @llvm.aarch64.sme.readq.horiz.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x i1>, i32, i32)512declare <vscale x 4 x i32> @llvm.aarch64.sme.readq.horiz.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32, i32)513declare <vscale x 4 x float> @llvm.aarch64.sme.readq.horiz.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, i32, i32)514declare <vscale x 2 x i64> @llvm.aarch64.sme.readq.horiz.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i32, i32)515declare <vscale x 2 x double> @llvm.aarch64.sme.readq.horiz.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, i32, i32)516declare <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i32, i32)517declare <vscale x 8 x i16> @llvm.aarch64.sme.readq.vert.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i32, i32)518declare <vscale x 8 x half> @llvm.aarch64.sme.readq.vert.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, i32, i32)519declare <vscale x 8 x bfloat> @llvm.aarch64.sme.readq.vert.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x i1>, i32, i32)520declare <vscale x 4 x i32> @llvm.aarch64.sme.readq.vert.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32, i32)521declare <vscale x 4 x float> @llvm.aarch64.sme.readq.vert.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, i32, i32)522declare <vscale x 2 x i64> @llvm.aarch64.sme.readq.vert.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i32, i32)523declare <vscale x 2 x double> @llvm.aarch64.sme.readq.vert.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, i32, i32)524 525; ------------------------------------------------------------------------------526; Dummy external function to force code retention.527; ------------------------------------------------------------------------------528 529declare void @use(...)530