522 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming -verify-machineinstrs < %s | FileCheck %s3 4define void @insert_row_b(i32 %tileslice, <vscale x 16 x i1> %pg,5; CHECK-LABEL: insert_row_b:6; CHECK: // %bb.0:7; CHECK-NEXT: mov w12, w08; CHECK-NEXT: mov za0h.b[w12, 0], p0/m, z0.b9; CHECK-NEXT: mov za0h.b[w12, 2], p0/m, z1.b10; CHECK-NEXT: mov za0h.b[w12, 4], p0/m, z2.b11; CHECK-NEXT: mov za0h.b[w12, 6], p0/m, z3.b12; CHECK-NEXT: mov za0h.b[w12, 8], p0/m, z4.b13; CHECK-NEXT: mov za0h.b[w12, 10], p0/m, z5.b14; CHECK-NEXT: mov za0h.b[w12, 12], p0/m, z6.b15; CHECK-NEXT: mov za0h.b[w12, 14], p0/m, z7.b16; CHECK-NEXT: ret17 <vscale x 16 x i8> %z0, <vscale x 16 x i8> %z1,18 <vscale x 16 x i8> %z2, <vscale x 16 x i8> %z3,19 <vscale x 16 x i8> %z4, <vscale x 16 x i8> %z5,20 <vscale x 16 x i8> %z6, <vscale x 16 x i8> %z7) {21 call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 %tileslice, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %z0)22 %tileslice.2 = add i32 %tileslice, 223 call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 %tileslice.2, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %z1)24 %tileslice.4 = add i32 %tileslice, 425 call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 %tileslice.4, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %z2)26 %tileslice.6 = add i32 %tileslice, 627 call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 %tileslice.6, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %z3)28 %tileslice.8 = add i32 %tileslice, 829 call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 %tileslice.8, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %z4)30 %tileslice.10 = add i32 %tileslice, 1031 call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 %tileslice.10, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %z5)32 %tileslice.12 = add i32 %tileslice, 1233 call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 %tileslice.12, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %z6)34 %tileslice.14 = add i32 %tileslice, 1435 call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 %tileslice.14, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %z7)36 ret void37}38 39define void @insert_col_b(i32 %tileslice, <vscale x 16 x i1> %pg,40; CHECK-LABEL: insert_col_b:41; CHECK: // %bb.0:42; CHECK-NEXT: mov w12, w043; CHECK-NEXT: mov za0v.b[w12, 1], p0/m, z0.b44; CHECK-NEXT: mov za0v.b[w12, 3], p0/m, z1.b45; CHECK-NEXT: mov za0v.b[w12, 5], p0/m, z2.b46; CHECK-NEXT: mov za0v.b[w12, 7], p0/m, z3.b47; CHECK-NEXT: mov za0v.b[w12, 9], p0/m, z4.b48; CHECK-NEXT: mov za0v.b[w12, 11], p0/m, z5.b49; CHECK-NEXT: mov za0v.b[w12, 13], p0/m, z6.b50; CHECK-NEXT: mov za0v.b[w12, 15], p0/m, z7.b51; CHECK-NEXT: ret52 <vscale x 16 x i8> %z0, <vscale x 16 x i8> %z1,53 <vscale x 16 x i8> %z2, <vscale x 16 x i8> %z3,54 <vscale x 16 x i8> %z4, <vscale x 16 x i8> %z5,55 <vscale x 16 x i8> %z6, <vscale x 16 x i8> %z7) {56 %tileslice.1 = add i32 %tileslice, 157 call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 %tileslice.1, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %z0)58 %tileslice.3 = add i32 %tileslice, 359 call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 %tileslice.3, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %z1)60 %tileslice.5 = add i32 %tileslice, 561 call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 %tileslice.5, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %z2)62 %tileslice.7 = add i32 %tileslice, 763 call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 %tileslice.7, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %z3)64 %tileslice.9 = add i32 %tileslice, 965 call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 %tileslice.9, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %z4)66 %tileslice.11 = add i32 %tileslice, 1167 call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 %tileslice.11, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %z5)68 %tileslice.13 = add i32 %tileslice, 1369 call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 %tileslice.13, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %z6)70 %tileslice.15 = add i32 %tileslice, 1571 call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 %tileslice.15, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %z7)72 ret void73}74 75define void @insert_row_h(i32 %tileslice, <vscale x 8 x i1> %pg,76; CHECK-LABEL: insert_row_h:77; CHECK: // %bb.0:78; CHECK-NEXT: mov w12, w079; CHECK-NEXT: mov za0h.h[w12, 0], p0/m, z0.h80; CHECK-NEXT: mov za0h.h[w12, 2], p0/m, z2.h81; CHECK-NEXT: mov za0h.h[w12, 4], p0/m, z4.h82; CHECK-NEXT: mov za0h.h[w12, 6], p0/m, z6.h83; CHECK-NEXT: ret84 <vscale x 8 x i16> %z0, <vscale x 8 x i16> %z1,85 <vscale x 8 x i16> %z2, <vscale x 8 x i16> %z3,86 <vscale x 8 x i16> %z4, <vscale x 8 x i16> %z5,87 <vscale x 8 x i16> %z6, <vscale x 8 x i16> %z7) {88 call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 0, i32 %tileslice, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %z0)89 %tileslice.2 = add i32 %tileslice, 290 call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 0, i32 %tileslice.2, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %z2)91 %tileslice.4 = add i32 %tileslice, 492 call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 0, i32 %tileslice.4, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %z4)93 %tileslice.6 = add i32 %tileslice, 694 call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 0, i32 %tileslice.6, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %z6)95 ret void96}97 98define void @insert_col_h(i32 %tileslice, <vscale x 8 x i1> %pg,99; CHECK-LABEL: insert_col_h:100; CHECK: // %bb.0:101; CHECK-NEXT: mov w12, w0102; CHECK-NEXT: mov za1v.h[w12, 1], p0/m, z1.h103; CHECK-NEXT: mov za1v.h[w12, 3], p0/m, z3.h104; CHECK-NEXT: mov za1v.h[w12, 5], p0/m, z5.h105; CHECK-NEXT: mov za1v.h[w12, 7], p0/m, z7.h106; CHECK-NEXT: ret107 <vscale x 8 x i16> %z0, <vscale x 8 x i16> %z1,108 <vscale x 8 x i16> %z2, <vscale x 8 x i16> %z3,109 <vscale x 8 x i16> %z4, <vscale x 8 x i16> %z5,110 <vscale x 8 x i16> %z6, <vscale x 8 x i16> %z7) {111 %tileslice.1 = add i32 %tileslice, 1112 call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 1, i32 %tileslice.1, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %z1)113 %tileslice.3 = add i32 %tileslice, 3114 call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 1, i32 %tileslice.3, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %z3)115 %tileslice.5 = add i32 %tileslice, 5116 call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 1, i32 %tileslice.5, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %z5)117 %tileslice.7 = add i32 %tileslice, 7118 call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 1, i32 %tileslice.7, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %z7)119 ret void120}121 122define void @insert_f16(i32 %tileslice, <vscale x 8 x i1> %pg,123; CHECK-LABEL: insert_f16:124; CHECK: // %bb.0:125; CHECK-NEXT: mov w12, w0126; CHECK-NEXT: mov za0h.h[w12, 0], p0/m, z0.h127; CHECK-NEXT: mov za0h.h[w12, 1], p0/m, z1.h128; CHECK-NEXT: mov za0v.h[w12, 2], p0/m, z2.h129; CHECK-NEXT: mov za0v.h[w12, 3], p0/m, z3.h130; CHECK-NEXT: mov za0h.h[w12, 4], p0/m, z4.h131; CHECK-NEXT: mov za0h.h[w12, 5], p0/m, z5.h132; CHECK-NEXT: mov za0v.h[w12, 6], p0/m, z6.h133; CHECK-NEXT: mov za0v.h[w12, 7], p0/m, z7.h134; CHECK-NEXT: ret135 <vscale x 8 x half> %z0, <vscale x 8 x half> %z1,136 <vscale x 8 x half> %z2, <vscale x 8 x half> %z3,137 <vscale x 8 x half> %z4, <vscale x 8 x half> %z5,138 <vscale x 8 x half> %z6, <vscale x 8 x half> %z7) {139 call void @llvm.aarch64.sme.write.horiz.nxv8f16(i32 0, i32 %tileslice, <vscale x 8 x i1> %pg, <vscale x 8 x half> %z0)140 %tileslice.1 = add i32 %tileslice, 1141 call void @llvm.aarch64.sme.write.horiz.nxv8f16(i32 0, i32 %tileslice.1, <vscale x 8 x i1> %pg, <vscale x 8 x half> %z1)142 %tileslice.2 = add i32 %tileslice, 2143 call void @llvm.aarch64.sme.write.vert.nxv8f16(i32 0, i32 %tileslice.2, <vscale x 8 x i1> %pg, <vscale x 8 x half> %z2)144 %tileslice.3 = add i32 %tileslice, 3145 call void @llvm.aarch64.sme.write.vert.nxv8f16(i32 0, i32 %tileslice.3, <vscale x 8 x i1> %pg, <vscale x 8 x half> %z3)146 %tileslice.4 = add i32 %tileslice, 4147 call void @llvm.aarch64.sme.write.horiz.nxv8f16(i32 0, i32 %tileslice.4, <vscale x 8 x i1> %pg, <vscale x 8 x half> %z4)148 %tileslice.5 = add i32 %tileslice, 5149 call void @llvm.aarch64.sme.write.horiz.nxv8f16(i32 0, i32 %tileslice.5, <vscale x 8 x i1> %pg, <vscale x 8 x half> %z5)150 %tileslice.6 = add i32 %tileslice, 6151 call void @llvm.aarch64.sme.write.vert.nxv8f16(i32 0, i32 %tileslice.6, <vscale x 8 x i1> %pg, <vscale x 8 x half> %z6)152 %tileslice.7 = add i32 %tileslice, 7153 call void @llvm.aarch64.sme.write.vert.nxv8f16(i32 0, i32 %tileslice.7, <vscale x 8 x i1> %pg, <vscale x 8 x half> %z7)154 ret void155}156 157define void @insert_bf16(i32 %tileslice, <vscale x 8 x i1> %pg,158; CHECK-LABEL: insert_bf16:159; CHECK: // %bb.0:160; CHECK-NEXT: mov w12, w0161; CHECK-NEXT: mov za0h.h[w12, 0], p0/m, z0.h162; CHECK-NEXT: mov za0h.h[w12, 1], p0/m, z1.h163; CHECK-NEXT: mov za0v.h[w12, 2], p0/m, z2.h164; CHECK-NEXT: mov za0v.h[w12, 3], p0/m, z3.h165; CHECK-NEXT: mov za0h.h[w12, 4], p0/m, z4.h166; CHECK-NEXT: mov za0h.h[w12, 5], p0/m, z5.h167; CHECK-NEXT: mov za0v.h[w12, 6], p0/m, z6.h168; CHECK-NEXT: mov za0v.h[w12, 7], p0/m, z7.h169; CHECK-NEXT: ret170 <vscale x 8 x bfloat> %z0, <vscale x 8 x bfloat> %z1,171 <vscale x 8 x bfloat> %z2, <vscale x 8 x bfloat> %z3,172 <vscale x 8 x bfloat> %z4, <vscale x 8 x bfloat> %z5,173 <vscale x 8 x bfloat> %z6, <vscale x 8 x bfloat> %z7) {174 call void @llvm.aarch64.sme.write.horiz.nxv8bf16(i32 0, i32 %tileslice, <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %z0)175 %tileslice.1 = add i32 %tileslice, 1176 call void @llvm.aarch64.sme.write.horiz.nxv8bf16(i32 0, i32 %tileslice.1, <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %z1)177 %tileslice.2 = add i32 %tileslice, 2178 call void @llvm.aarch64.sme.write.vert.nxv8bf16(i32 0, i32 %tileslice.2, <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %z2)179 %tileslice.3 = add i32 %tileslice, 3180 call void @llvm.aarch64.sme.write.vert.nxv8bf16(i32 0, i32 %tileslice.3, <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %z3)181 %tileslice.4 = add i32 %tileslice, 4182 call void @llvm.aarch64.sme.write.horiz.nxv8bf16(i32 0, i32 %tileslice.4, <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %z4)183 %tileslice.5 = add i32 %tileslice, 5184 call void @llvm.aarch64.sme.write.horiz.nxv8bf16(i32 0, i32 %tileslice.5, <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %z5)185 %tileslice.6 = add i32 %tileslice, 6186 call void @llvm.aarch64.sme.write.vert.nxv8bf16(i32 0, i32 %tileslice.6, <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %z6)187 %tileslice.7 = add i32 %tileslice, 7188 call void @llvm.aarch64.sme.write.vert.nxv8bf16(i32 0, i32 %tileslice.7, <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %z7)189 ret void190}191 192define void @insert_row_s(i32 %tileslice, <vscale x 4 x i1> %pg,193; CHECK-LABEL: insert_row_s:194; CHECK: // %bb.0:195; CHECK-NEXT: mov w12, w0196; CHECK-NEXT: mov za0h.s[w12, 0], p0/m, z0.s197; CHECK-NEXT: mov za0h.s[w12, 2], p0/m, z2.s198; CHECK-NEXT: ret199 <vscale x 4 x i32> %z0, <vscale x 4 x i32> %z1,200 <vscale x 4 x i32> %z2, <vscale x 4 x i32> %z3) {201 call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 0, i32 %tileslice, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %z0)202 %tileslice.2 = add i32 %tileslice, 2203 call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 0, i32 %tileslice.2, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %z2)204 ret void205}206 207define void @insert_col_s(i32 %tileslice, <vscale x 4 x i1> %pg,208; CHECK-LABEL: insert_col_s:209; CHECK: // %bb.0:210; CHECK-NEXT: mov w12, w0211; CHECK-NEXT: mov za3v.s[w12, 1], p0/m, z1.s212; CHECK-NEXT: mov za3v.s[w12, 3], p0/m, z3.s213; CHECK-NEXT: ret214 <vscale x 4 x i32> %z0, <vscale x 4 x i32> %z1,215 <vscale x 4 x i32> %z2, <vscale x 4 x i32> %z3) {216 %tileslice.1 = add i32 %tileslice, 1217 call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 3, i32 %tileslice.1, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %z1)218 %tileslice.3 = add i32 %tileslice, 3219 call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 3, i32 %tileslice.3, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %z3)220 ret void221}222 223define void @insert_f32(i32 %tileslice, <vscale x 4 x i1> %pg,224; CHECK-LABEL: insert_f32:225; CHECK: // %bb.0:226; CHECK-NEXT: mov w12, w0227; CHECK-NEXT: mov za0h.s[w12, 0], p0/m, z0.s228; CHECK-NEXT: mov za0h.s[w12, 1], p0/m, z1.s229; CHECK-NEXT: mov za0v.s[w12, 2], p0/m, z2.s230; CHECK-NEXT: mov za0v.s[w12, 3], p0/m, z3.s231; CHECK-NEXT: ret232 <vscale x 4 x float> %z0, <vscale x 4 x float> %z1,233 <vscale x 4 x float> %z2, <vscale x 4 x float> %z3) {234 call void @llvm.aarch64.sme.write.horiz.nxv4f32(i32 0, i32 %tileslice, <vscale x 4 x i1> %pg, <vscale x 4 x float> %z0)235 %tileslice.1 = add i32 %tileslice, 1236 call void @llvm.aarch64.sme.write.horiz.nxv4f32(i32 0, i32 %tileslice.1, <vscale x 4 x i1> %pg, <vscale x 4 x float> %z1)237 %tileslice.2 = add i32 %tileslice, 2238 call void @llvm.aarch64.sme.write.vert.nxv4f32(i32 0, i32 %tileslice.2, <vscale x 4 x i1> %pg, <vscale x 4 x float> %z2)239 %tileslice.3 = add i32 %tileslice, 3240 call void @llvm.aarch64.sme.write.vert.nxv4f32(i32 0, i32 %tileslice.3, <vscale x 4 x i1> %pg, <vscale x 4 x float> %z3)241 ret void242}243 244define void @insert_row_d(i32 %tileslice, <vscale x 2 x i1> %pg,245; CHECK-LABEL: insert_row_d:246; CHECK: // %bb.0:247; CHECK-NEXT: mov w12, w0248; CHECK-NEXT: mov za0h.d[w12, 0], p0/m, z0.d249; CHECK-NEXT: ret250 <vscale x 2 x i64> %z0, <vscale x 2 x i64> %z1) {251 call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 0, i32 %tileslice, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %z0)252 ret void253}254 255define void @insert_col_d(i32 %tileslice, <vscale x 2 x i1> %pg,256; CHECK-LABEL: insert_col_d:257; CHECK: // %bb.0:258; CHECK-NEXT: mov w12, w0259; CHECK-NEXT: mov za7v.d[w12, 1], p0/m, z1.d260; CHECK-NEXT: ret261 <vscale x 2 x i64> %z0, <vscale x 2 x i64> %z1) {262 %tileslice.1 = add i32 %tileslice, 1263 call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 7, i32 %tileslice.1, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %z1)264 ret void265}266 267define void @insert_f64(i32 %tileslice, <vscale x 2 x i1> %pg,268; CHECK-LABEL: insert_f64:269; CHECK: // %bb.0:270; CHECK-NEXT: mov w12, w0271; CHECK-NEXT: mov za0h.d[w12, 0], p0/m, z0.d272; CHECK-NEXT: mov za0v.d[w12, 1], p0/m, z1.d273; CHECK-NEXT: ret274 <vscale x 2 x double> %z0, <vscale x 2 x double> %z1) {275 call void @llvm.aarch64.sme.write.horiz.nxv2f64(i32 0, i32 %tileslice, <vscale x 2 x i1> %pg, <vscale x 2 x double> %z0)276 %tileslice.1 = add i32 %tileslice, 1277 call void @llvm.aarch64.sme.write.vert.nxv2f64(i32 0, i32 %tileslice.1, <vscale x 2 x i1> %pg, <vscale x 2 x double> %z1)278 ret void279}280 281define void @insert_row_q_v16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %zn) {282; CHECK-LABEL: insert_row_q_v16i8:283; CHECK: // %bb.0:284; CHECK-NEXT: mov w12, wzr285; CHECK-NEXT: mov za0h.q[w12, 0], p0/m, z0.q286; CHECK-NEXT: ret287 call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 0, i32 0, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %zn)288 ret void289}290 291define void @insert_row_q_v8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %zn) {292; CHECK-LABEL: insert_row_q_v8i16:293; CHECK: // %bb.0:294; CHECK-NEXT: mov w12, wzr295; CHECK-NEXT: mov za0h.q[w12, 0], p0/m, z0.q296; CHECK-NEXT: ret297 call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 0, i32 0, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %zn)298 ret void299}300 301define void @insert_row_q_v8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %zn) {302; CHECK-LABEL: insert_row_q_v8f16:303; CHECK: // %bb.0:304; CHECK-NEXT: mov w12, wzr305; CHECK-NEXT: mov za0h.q[w12, 0], p0/m, z0.q306; CHECK-NEXT: ret307 call void @llvm.aarch64.sme.writeq.horiz.nxv8f16(i32 0, i32 0, <vscale x 8 x i1> %pg, <vscale x 8 x half> %zn)308 ret void309}310 311define void @insert_row_q_v8bf16(<vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %zn) {312; CHECK-LABEL: insert_row_q_v8bf16:313; CHECK: // %bb.0:314; CHECK-NEXT: mov w12, wzr315; CHECK-NEXT: mov za0h.q[w12, 0], p0/m, z0.q316; CHECK-NEXT: ret317 call void @llvm.aarch64.sme.writeq.horiz.nxv8bf16(i32 0, i32 0, <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %zn)318 ret void319}320 321define void @insert_row_q_v4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %zn) {322; CHECK-LABEL: insert_row_q_v4i32:323; CHECK: // %bb.0:324; CHECK-NEXT: mov w12, wzr325; CHECK-NEXT: mov za0h.q[w12, 0], p0/m, z0.q326; CHECK-NEXT: ret327 call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 0, i32 0, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %zn)328 ret void329}330 331define void @insert_row_q_v4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %zn) {332; CHECK-LABEL: insert_row_q_v4f32:333; CHECK: // %bb.0:334; CHECK-NEXT: mov w12, wzr335; CHECK-NEXT: mov za0h.q[w12, 0], p0/m, z0.q336; CHECK-NEXT: ret337 call void @llvm.aarch64.sme.writeq.horiz.nxv4f32(i32 0, i32 0, <vscale x 4 x i1> %pg, <vscale x 4 x float> %zn)338 ret void339}340 341define void @insert_row_q_v2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %zn) {342; CHECK-LABEL: insert_row_q_v2i64:343; CHECK: // %bb.0:344; CHECK-NEXT: mov w12, wzr345; CHECK-NEXT: mov za0h.q[w12, 0], p0/m, z0.q346; CHECK-NEXT: ret347 call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 0, i32 0, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %zn)348 ret void349}350 351define void @insert_row_q_v2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %zn) {352; CHECK-LABEL: insert_row_q_v2f64:353; CHECK: // %bb.0:354; CHECK-NEXT: mov w12, wzr355; CHECK-NEXT: mov za0h.q[w12, 0], p0/m, z0.q356; CHECK-NEXT: ret357 call void @llvm.aarch64.sme.writeq.horiz.nxv2f64(i32 0, i32 0, <vscale x 2 x i1> %pg, <vscale x 2 x double> %zn)358 ret void359}360 361define void @insert_col_q_v16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %zn) {362; CHECK-LABEL: insert_col_q_v16i8:363; CHECK: // %bb.0:364; CHECK-NEXT: mov w12, wzr365; CHECK-NEXT: mov za15v.q[w12, 0], p0/m, z0.q366; CHECK-NEXT: ret367 call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 15, i32 0, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %zn)368 ret void369}370 371define void @insert_col_q_v8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %zn) {372; CHECK-LABEL: insert_col_q_v8i16:373; CHECK: // %bb.0:374; CHECK-NEXT: mov w12, wzr375; CHECK-NEXT: mov za15v.q[w12, 0], p0/m, z0.q376; CHECK-NEXT: ret377 call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 15, i32 0, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %zn)378 ret void379}380 381define void @insert_col_q_v8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %zn) {382; CHECK-LABEL: insert_col_q_v8f16:383; CHECK: // %bb.0:384; CHECK-NEXT: mov w12, wzr385; CHECK-NEXT: mov za15v.q[w12, 0], p0/m, z0.q386; CHECK-NEXT: ret387 call void @llvm.aarch64.sme.writeq.vert.nxv8f16(i32 15, i32 0, <vscale x 8 x i1> %pg, <vscale x 8 x half> %zn)388 ret void389}390 391define void @insert_col_q_v8bf16(<vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %zn) {392; CHECK-LABEL: insert_col_q_v8bf16:393; CHECK: // %bb.0:394; CHECK-NEXT: mov w12, wzr395; CHECK-NEXT: mov za15v.q[w12, 0], p0/m, z0.q396; CHECK-NEXT: ret397 call void @llvm.aarch64.sme.writeq.vert.nxv8bf16(i32 15, i32 0, <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %zn)398 ret void399}400 401define void @insert_col_q_v4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %zn) {402; CHECK-LABEL: insert_col_q_v4i32:403; CHECK: // %bb.0:404; CHECK-NEXT: mov w12, wzr405; CHECK-NEXT: mov za15v.q[w12, 0], p0/m, z0.q406; CHECK-NEXT: ret407 call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 15, i32 0, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %zn)408 ret void409}410 411define void @insert_col_q_v4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %zn) {412; CHECK-LABEL: insert_col_q_v4f32:413; CHECK: // %bb.0:414; CHECK-NEXT: mov w12, wzr415; CHECK-NEXT: mov za15v.q[w12, 0], p0/m, z0.q416; CHECK-NEXT: ret417 call void @llvm.aarch64.sme.writeq.vert.nxv4f32(i32 15, i32 0, <vscale x 4 x i1> %pg, <vscale x 4 x float> %zn)418 ret void419}420 421define void @insert_col_q_v2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %zn) {422; CHECK-LABEL: insert_col_q_v2i64:423; CHECK: // %bb.0:424; CHECK-NEXT: mov w12, wzr425; CHECK-NEXT: mov za15v.q[w12, 0], p0/m, z0.q426; CHECK-NEXT: ret427 call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 15, i32 0, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %zn)428 ret void429}430 431define void @insert_col_q_v2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %zn) {432; CHECK-LABEL: insert_col_q_v2f64:433; CHECK: // %bb.0:434; CHECK-NEXT: mov w12, wzr435; CHECK-NEXT: mov za15v.q[w12, 0], p0/m, z0.q436; CHECK-NEXT: ret437 call void @llvm.aarch64.sme.writeq.vert.nxv2f64(i32 15, i32 0, <vscale x 2 x i1> %pg, <vscale x 2 x double> %zn)438 ret void439}440 441define void @test_sink_offset_operand(<vscale x 4 x i1> %pg, i32 %base, i32 %N) {442; CHECK-LABEL: test_sink_offset_operand:443; CHECK: // %bb.0: // %entry444; CHECK-NEXT: mov z0.s, #0 // =0x0445; CHECK-NEXT: mov w12, w0446; CHECK-NEXT: .LBB28_1: // %for.body447; CHECK-NEXT: // =>This Inner Loop Header: Depth=1448; CHECK-NEXT: mov za0h.s[w12, 0], p0/m, z0.s449; CHECK-NEXT: subs w1, w1, #3450; CHECK-NEXT: mov za0h.s[w12, 1], p0/m, z0.s451; CHECK-NEXT: mov za0h.s[w12, 2], p0/m, z0.s452; CHECK-NEXT: b.ne .LBB28_1453; CHECK-NEXT: // %bb.2: // %exit454; CHECK-NEXT: ret455entry:456 %add1 = add i32 %base, 1457 %add2 = add i32 %base, 2458 br label %for.body459 460for.body:461 %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]462 call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 0, i32 %base, <vscale x 4 x i1> %pg, <vscale x 4 x i32> zeroinitializer)463 call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 0, i32 %add1, <vscale x 4 x i1> %pg, <vscale x 4 x i32> zeroinitializer)464 call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 0, i32 %add2, <vscale x 4 x i1> %pg, <vscale x 4 x i32> zeroinitializer)465 %inc = add nuw nsw i32 %i, 3466 %exitcond.not = icmp eq i32 %inc, %N467 br i1 %exitcond.not, label %exit, label %for.body468 469exit:470 ret void471}472 473define void @test_add_with_disjoint_or(i64 %idx, <vscale x 4 x i1> %pg) {474; CHECK-LABEL: test_add_with_disjoint_or:475; CHECK: // %bb.0:476; CHECK-NEXT: mov z0.s, #0 // =0x0477; CHECK-NEXT: mov x12, x0478; CHECK-NEXT: mov za0h.s[w12, 0], p0/m, z0.s479; CHECK-NEXT: mov za0h.s[w12, 1], p0/m, z0.s480; CHECK-NEXT: ret481 %idx.trunc = trunc i64 %idx to i32482 call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 0, i32 %idx.trunc, <vscale x 4 x i1> %pg, <vscale x 4 x i32> zeroinitializer)483 %idx2 = or disjoint i64 %idx, 1484 %idx2.trunc = trunc i64 %idx2 to i32485 call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 0, i32 %idx2.trunc, <vscale x 4 x i1> %pg, <vscale x 4 x i32> zeroinitializer)486 ret void487}488 489declare void @llvm.aarch64.sme.write.horiz.nxv16i8(i32, i32, <vscale x 16 x i1>, <vscale x 16 x i8>)490declare void @llvm.aarch64.sme.write.horiz.nxv8i16(i32, i32, <vscale x 8 x i1>, <vscale x 8 x i16>)491declare void @llvm.aarch64.sme.write.horiz.nxv8f16(i32, i32, <vscale x 8 x i1>, <vscale x 8 x half>)492declare void @llvm.aarch64.sme.write.horiz.nxv8bf16(i32, i32, <vscale x 8 x i1>, <vscale x 8 x bfloat>)493declare void @llvm.aarch64.sme.write.horiz.nxv4i32(i32, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)494declare void @llvm.aarch64.sme.write.horiz.nxv4f32(i32, i32, <vscale x 4 x i1>, <vscale x 4 x float>)495declare void @llvm.aarch64.sme.write.horiz.nxv2i64(i32, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)496declare void @llvm.aarch64.sme.write.horiz.nxv2f64(i32, i32, <vscale x 2 x i1>, <vscale x 2 x double>)497declare void @llvm.aarch64.sme.write.vert.nxv16i8(i32, i32, <vscale x 16 x i1>, <vscale x 16 x i8>)498declare void @llvm.aarch64.sme.write.vert.nxv8i16(i32, i32, <vscale x 8 x i1>, <vscale x 8 x i16>)499declare void @llvm.aarch64.sme.write.vert.nxv8f16(i32, i32, <vscale x 8 x i1>, <vscale x 8 x half>)500declare void @llvm.aarch64.sme.write.vert.nxv8bf16(i32, i32, <vscale x 8 x i1>, <vscale x 8 x bfloat>)501declare void @llvm.aarch64.sme.write.vert.nxv4i32(i32, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)502declare void @llvm.aarch64.sme.write.vert.nxv4f32(i32, i32, <vscale x 4 x i1>, <vscale x 4 x float>)503declare void @llvm.aarch64.sme.write.vert.nxv2i64(i32, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)504declare void @llvm.aarch64.sme.write.vert.nxv2f64(i32, i32, <vscale x 2 x i1>, <vscale x 2 x double>)505 506declare void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32, i32, <vscale x 16 x i1>, <vscale x 16 x i8>)507declare void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32, i32, <vscale x 8 x i1>, <vscale x 8 x i16>)508declare void @llvm.aarch64.sme.writeq.horiz.nxv8f16(i32, i32, <vscale x 8 x i1>, <vscale x 8 x half>)509declare void @llvm.aarch64.sme.writeq.horiz.nxv8bf16(i32, i32, <vscale x 8 x i1>, <vscale x 8 x bfloat>)510declare void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)511declare void @llvm.aarch64.sme.writeq.horiz.nxv4f32(i32, i32, <vscale x 4 x i1>, <vscale x 4 x float>)512declare void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)513declare void @llvm.aarch64.sme.writeq.horiz.nxv2f64(i32, i32, <vscale x 2 x i1>, <vscale x 2 x double>)514declare void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32, i32, <vscale x 16 x i1>, <vscale x 16 x i8>)515declare void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32, i32, <vscale x 8 x i1>, <vscale x 8 x i16>)516declare void @llvm.aarch64.sme.writeq.vert.nxv8f16(i32, i32, <vscale x 8 x i1>, <vscale x 8 x half>)517declare void @llvm.aarch64.sme.writeq.vert.nxv8bf16(i32, i32, <vscale x 8 x i1>, <vscale x 8 x bfloat>)518declare void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)519declare void @llvm.aarch64.sme.writeq.vert.nxv4f32(i32, i32, <vscale x 4 x i1>, <vscale x 4 x float>)520declare void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)521declare void @llvm.aarch64.sme.writeq.vert.nxv2f64(i32, i32, <vscale x 2 x i1>, <vscale x 2 x double>)522