528 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming -verify-machineinstrs < %s | FileCheck %s3 4define void @ld1b(<vscale x 16 x i1> %pg, ptr %ptr, i32 %sliceidx) {5; CHECK-LABEL: ld1b:6; CHECK: // %bb.0:7; CHECK-NEXT: mov w12, w18; CHECK-NEXT: mov w13, wzr9; CHECK-NEXT: ld1b {za0h.b[w12, 15]}, p0/z, [x0]10; CHECK-NEXT: ld1b {za0v.b[w13, 0]}, p0/z, [x0]11; CHECK-NEXT: ret12 %tileslice = add i32 %sliceidx, 1513 call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> %pg, ptr %ptr, i32 0, i32 %tileslice)14 call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> %pg, ptr %ptr, i32 0, i32 0)15 ret void;16}17 18define void @ld1b_with_addr_offset(<vscale x 16 x i1> %pg, ptr %ptr, i64 %index, i32 %sliceidx) {19; CHECK-LABEL: ld1b_with_addr_offset:20; CHECK: // %bb.0:21; CHECK-NEXT: mov w13, wzr22; CHECK-NEXT: mov w12, w223; CHECK-NEXT: ld1b {za0h.b[w13, 0]}, p0/z, [x0, x1]24; CHECK-NEXT: ld1b {za0v.b[w12, 15]}, p0/z, [x0, x1]25; CHECK-NEXT: ret26 %base = getelementptr i8, ptr %ptr, i64 %index27 %tileslice = add i32 %sliceidx, 1528 call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> %pg, ptr %base, i32 0, i32 0)29 call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> %pg, ptr %base, i32 0, i32 %tileslice)30 ret void;31}32 33define void @ld1h(<vscale x 8 x i1> %pg, ptr %ptr, i32 %sliceidx) {34; CHECK-LABEL: ld1h:35; CHECK: // %bb.0:36; CHECK-NEXT: mov w12, w137; CHECK-NEXT: mov w13, wzr38; CHECK-NEXT: ld1h {za0h.h[w12, 7]}, p0/z, [x0]39; CHECK-NEXT: ld1h {za1h.h[w13, 0]}, p0/z, [x0]40; CHECK-NEXT: ld1h {za0v.h[w13, 0]}, p0/z, [x0]41; CHECK-NEXT: ld1h {za1v.h[w12, 7]}, p0/z, [x0]42; CHECK-NEXT: ret43 %tileslice = add i32 %sliceidx, 744 call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> %pg, ptr %ptr, i32 0, i32 %tileslice)45 call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> %pg, ptr %ptr, i32 1, i32 0)46 call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> %pg, ptr %ptr, i32 0, i32 0)47 call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> %pg, ptr %ptr, i32 1, i32 %tileslice)48 ret void;49}50 51define void @ld1h_with_addr_offset(<vscale x 8 x i1> %pg, ptr %ptr, i64 %index, i32 %sliceidx) {52; CHECK-LABEL: ld1h_with_addr_offset:53; CHECK: // %bb.0:54; CHECK-NEXT: mov w12, w255; CHECK-NEXT: mov w13, wzr56; CHECK-NEXT: ld1h {za0h.h[w12, 7]}, p0/z, [x0, x1, lsl #1]57; CHECK-NEXT: ld1h {za1v.h[w13, 0]}, p0/z, [x0, x1, lsl #1]58; CHECK-NEXT: ret59 %base = getelementptr i16, ptr %ptr, i64 %index60 %tileslice = add i32 %sliceidx, 761 call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> %pg, ptr %base, i32 0, i32 %tileslice)62 call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> %pg, ptr %base, i32 1, i32 0)63 ret void;64}65 66define void @ld1w(<vscale x 4 x i1> %pg, ptr %ptr, i32 %sliceidx) {67; CHECK-LABEL: ld1w:68; CHECK: // %bb.0:69; CHECK-NEXT: mov w12, w170; CHECK-NEXT: mov w13, wzr71; CHECK-NEXT: ld1w {za0h.s[w13, 0]}, p0/z, [x0]72; CHECK-NEXT: ld1w {za1h.s[w13, 0]}, p0/z, [x0]73; CHECK-NEXT: ld1w {za2h.s[w13, 0]}, p0/z, [x0]74; CHECK-NEXT: ld1w {za3h.s[w12, 3]}, p0/z, [x0]75; CHECK-NEXT: ld1w {za0v.s[w13, 0]}, p0/z, [x0]76; CHECK-NEXT: ld1w {za1v.s[w13, 0]}, p0/z, [x0]77; CHECK-NEXT: ld1w {za2v.s[w12, 3]}, p0/z, [x0]78; CHECK-NEXT: ld1w {za3v.s[w13, 0]}, p0/z, [x0]79; CHECK-NEXT: ret80 %tileslice = add i32 %sliceidx, 381 call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> %pg, ptr %ptr, i32 0, i32 0)82 call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> %pg, ptr %ptr, i32 1, i32 0)83 call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> %pg, ptr %ptr, i32 2, i32 0)84 call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> %pg, ptr %ptr, i32 3, i32 %tileslice)85 call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> %pg, ptr %ptr, i32 0, i32 0)86 call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> %pg, ptr %ptr, i32 1, i32 0)87 call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> %pg, ptr %ptr, i32 2, i32 %tileslice)88 call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> %pg, ptr %ptr, i32 3, i32 0)89 ret void;90}91 92define void @ld1w_with_addr_offset(<vscale x 4 x i1> %pg, ptr %ptr, i64 %index, i32 %sliceidx) {93; CHECK-LABEL: ld1w_with_addr_offset:94; CHECK: // %bb.0:95; CHECK-NEXT: mov w12, w296; CHECK-NEXT: mov w13, wzr97; CHECK-NEXT: ld1w {za0h.s[w13, 0]}, p0/z, [x0, x1, lsl #2]98; CHECK-NEXT: ld1w {za3v.s[w12, 3]}, p0/z, [x0, x1, lsl #2]99; CHECK-NEXT: ret100 %base = getelementptr i32, ptr %ptr, i64 %index101 %tileslice = add i32 %sliceidx, 3102 call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> %pg, ptr %base, i32 0, i32 0)103 call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> %pg, ptr %base, i32 3, i32 %tileslice)104 ret void;105}106 107define void @ld1d(<vscale x 2 x i1> %pg, ptr %ptr, i32 %sliceidx) {108; CHECK-LABEL: ld1d:109; CHECK: // %bb.0:110; CHECK-NEXT: mov w12, w1111; CHECK-NEXT: mov w13, wzr112; CHECK-NEXT: ld1d {za0h.d[w13, 0]}, p0/z, [x0]113; CHECK-NEXT: ld1d {za1h.d[w13, 0]}, p0/z, [x0]114; CHECK-NEXT: ld1d {za2h.d[w13, 0]}, p0/z, [x0]115; CHECK-NEXT: ld1d {za3h.d[w13, 0]}, p0/z, [x0]116; CHECK-NEXT: ld1d {za4h.d[w12, 1]}, p0/z, [x0]117; CHECK-NEXT: ld1d {za5h.d[w13, 0]}, p0/z, [x0]118; CHECK-NEXT: ld1d {za6h.d[w13, 0]}, p0/z, [x0]119; CHECK-NEXT: ld1d {za7h.d[w13, 0]}, p0/z, [x0]120; CHECK-NEXT: ld1d {za0v.d[w13, 0]}, p0/z, [x0]121; CHECK-NEXT: ld1d {za1v.d[w13, 0]}, p0/z, [x0]122; CHECK-NEXT: ld1d {za2v.d[w13, 0]}, p0/z, [x0]123; CHECK-NEXT: ld1d {za3v.d[w13, 0]}, p0/z, [x0]124; CHECK-NEXT: ld1d {za4v.d[w13, 0]}, p0/z, [x0]125; CHECK-NEXT: ld1d {za5v.d[w13, 0]}, p0/z, [x0]126; CHECK-NEXT: ld1d {za6v.d[w13, 0]}, p0/z, [x0]127; CHECK-NEXT: ld1d {za7v.d[w12, 1]}, p0/z, [x0]128; CHECK-NEXT: ret129 %tileslice = add i32 %sliceidx, 1130 call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i32 0, i32 0)131 call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i32 1, i32 0)132 call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i32 2, i32 0)133 call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i32 3, i32 0)134 call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i32 4, i32 %tileslice)135 call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i32 5, i32 0)136 call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i32 6, i32 0)137 call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %ptr, i32 7, i32 0)138 call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i32 0, i32 0)139 call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i32 1, i32 0)140 call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i32 2, i32 0)141 call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i32 3, i32 0)142 call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i32 4, i32 0)143 call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i32 5, i32 0)144 call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i32 6, i32 0)145 call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %ptr, i32 7, i32 %tileslice)146 ret void;147}148 149define void @ld1d_with_addr_offset(<vscale x 2 x i1> %pg, ptr %ptr, i64 %index, i32 %sliceidx) {150; CHECK-LABEL: ld1d_with_addr_offset:151; CHECK: // %bb.0:152; CHECK-NEXT: mov w12, w2153; CHECK-NEXT: mov w13, wzr154; CHECK-NEXT: ld1d {za0h.d[w12, 1]}, p0/z, [x0, x1, lsl #3]155; CHECK-NEXT: ld1d {za7v.d[w13, 0]}, p0/z, [x0, x1, lsl #3]156; CHECK-NEXT: ret157 %base = getelementptr i64, ptr %ptr, i64 %index158 %tileslice = add i32 %sliceidx, 1159 call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> %pg, ptr %base, i32 0, i32 %tileslice)160 call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> %pg, ptr %base, i32 7, i32 0)161 ret void;162}163 164define void @ld1q(<vscale x 1 x i1> %pg, ptr %ptr) {165; CHECK-LABEL: ld1q:166; CHECK: // %bb.0:167; CHECK-NEXT: mov w12, wzr168; CHECK-NEXT: ld1q {za0h.q[w12, 0]}, p0/z, [x0]169; CHECK-NEXT: ld1q {za1h.q[w12, 0]}, p0/z, [x0]170; CHECK-NEXT: ld1q {za2h.q[w12, 0]}, p0/z, [x0]171; CHECK-NEXT: ld1q {za3h.q[w12, 0]}, p0/z, [x0]172; CHECK-NEXT: ld1q {za4h.q[w12, 0]}, p0/z, [x0]173; CHECK-NEXT: ld1q {za5h.q[w12, 0]}, p0/z, [x0]174; CHECK-NEXT: ld1q {za6h.q[w12, 0]}, p0/z, [x0]175; CHECK-NEXT: ld1q {za7h.q[w12, 0]}, p0/z, [x0]176; CHECK-NEXT: ld1q {za8h.q[w12, 0]}, p0/z, [x0]177; CHECK-NEXT: ld1q {za9h.q[w12, 0]}, p0/z, [x0]178; CHECK-NEXT: ld1q {za10h.q[w12, 0]}, p0/z, [x0]179; CHECK-NEXT: ld1q {za11h.q[w12, 0]}, p0/z, [x0]180; CHECK-NEXT: ld1q {za12h.q[w12, 0]}, p0/z, [x0]181; CHECK-NEXT: ld1q {za13h.q[w12, 0]}, p0/z, [x0]182; CHECK-NEXT: ld1q {za14h.q[w12, 0]}, p0/z, [x0]183; CHECK-NEXT: ld1q {za15h.q[w12, 0]}, p0/z, [x0]184; CHECK-NEXT: ld1q {za0v.q[w12, 0]}, p0/z, [x0]185; CHECK-NEXT: ld1q {za1v.q[w12, 0]}, p0/z, [x0]186; CHECK-NEXT: ld1q {za2v.q[w12, 0]}, p0/z, [x0]187; CHECK-NEXT: ld1q {za3v.q[w12, 0]}, p0/z, [x0]188; CHECK-NEXT: ld1q {za4v.q[w12, 0]}, p0/z, [x0]189; CHECK-NEXT: ld1q {za5v.q[w12, 0]}, p0/z, [x0]190; CHECK-NEXT: ld1q {za6v.q[w12, 0]}, p0/z, [x0]191; CHECK-NEXT: ld1q {za7v.q[w12, 0]}, p0/z, [x0]192; CHECK-NEXT: ld1q {za8v.q[w12, 0]}, p0/z, [x0]193; CHECK-NEXT: ld1q {za9v.q[w12, 0]}, p0/z, [x0]194; CHECK-NEXT: ld1q {za10v.q[w12, 0]}, p0/z, [x0]195; CHECK-NEXT: ld1q {za11v.q[w12, 0]}, p0/z, [x0]196; CHECK-NEXT: ld1q {za12v.q[w12, 0]}, p0/z, [x0]197; CHECK-NEXT: ld1q {za13v.q[w12, 0]}, p0/z, [x0]198; CHECK-NEXT: ld1q {za14v.q[w12, 0]}, p0/z, [x0]199; CHECK-NEXT: ld1q {za15v.q[w12, 0]}, p0/z, [x0]200; CHECK-NEXT: ret201 call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i32 0, i32 0)202 call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i32 1, i32 0)203 call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i32 2, i32 0)204 call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i32 3, i32 0)205 call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i32 4, i32 0)206 call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i32 5, i32 0)207 call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i32 6, i32 0)208 call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i32 7, i32 0)209 call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i32 8, i32 0)210 call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i32 9, i32 0)211 call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i32 10, i32 0)212 call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i32 11, i32 0)213 call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i32 12, i32 0)214 call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i32 13, i32 0)215 call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i32 14, i32 0)216 call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %ptr, i32 15, i32 0)217 call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i32 0, i32 0)218 call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i32 1, i32 0)219 call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i32 2, i32 0)220 call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i32 3, i32 0)221 call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i32 4, i32 0)222 call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i32 5, i32 0)223 call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i32 6, i32 0)224 call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i32 7, i32 0)225 call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i32 8, i32 0)226 call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i32 9, i32 0)227 call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i32 10, i32 0)228 call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i32 11, i32 0)229 call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i32 12, i32 0)230 call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i32 13, i32 0)231 call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i32 14, i32 0)232 call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %ptr, i32 15, i32 0)233 ret void;234}235 236define void @ld1q_with_addr_offset(<vscale x 1 x i1> %pg, ptr %ptr, i64 %index) {237; CHECK-LABEL: ld1q_with_addr_offset:238; CHECK: // %bb.0:239; CHECK-NEXT: mov w12, wzr240; CHECK-NEXT: ld1q {za0h.q[w12, 0]}, p0/z, [x0, x1, lsl #4]241; CHECK-NEXT: ld1q {za15v.q[w12, 0]}, p0/z, [x0, x1, lsl #4]242; CHECK-NEXT: ret243 %base = getelementptr i128, ptr %ptr, i64 %index244 call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> %pg, ptr %base, i32 0, i32 0)245 call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> %pg, ptr %base, i32 15, i32 0)246 ret void;247}248 249define void @ldr(ptr %ptr) {250; CHECK-LABEL: ldr:251; CHECK: // %bb.0:252; CHECK-NEXT: mov w12, wzr253; CHECK-NEXT: ldr za[w12, 0], [x0]254; CHECK-NEXT: ret255 call void @llvm.aarch64.sme.ldr(i32 0, ptr %ptr, i32 0)256 ret void;257}258 259define void @ldr_with_off_15(ptr %ptr) {260; CHECK-LABEL: ldr_with_off_15:261; CHECK: // %bb.0:262; CHECK-NEXT: mov w12, #15 // =0xf263; CHECK-NEXT: add x8, x0, #15264; CHECK-NEXT: ldr za[w12, 0], [x8]265; CHECK-NEXT: ret266 %base = getelementptr i8, ptr %ptr, i64 15267 call void @llvm.aarch64.sme.ldr(i32 15, ptr %base, i32 0)268 ret void;269}270 271define void @ldr_with_off_15mulvl(ptr %ptr) {272; CHECK-LABEL: ldr_with_off_15mulvl:273; CHECK: // %bb.0:274; CHECK-NEXT: mov w12, #15 // =0xf275; CHECK-NEXT: addvl x8, x0, #15276; CHECK-NEXT: ldr za[w12, 0], [x8]277; CHECK-NEXT: ret278 %vscale = call i64 @llvm.vscale.i64()279 %mulvl = mul i64 %vscale, 240280 %base = getelementptr i8, ptr %ptr, i64 %mulvl281 call void @llvm.aarch64.sme.ldr(i32 15, ptr %base, i32 0)282 ret void;283}284 285define void @ldr_with_off_16mulvl(ptr %ptr) {286; CHECK-LABEL: ldr_with_off_16mulvl:287; CHECK: // %bb.0:288; CHECK-NEXT: mov w12, #16 // =0x10289; CHECK-NEXT: addvl x8, x0, #16290; CHECK-NEXT: ldr za[w12, 0], [x8]291; CHECK-NEXT: ret292 %vscale = call i64 @llvm.vscale.i64()293 %mulvl = mul i64 %vscale, 256294 %base = getelementptr i8, ptr %ptr, i64 %mulvl295 call void @llvm.aarch64.sme.ldr(i32 16, ptr %base, i32 0)296 ret void;297}298 299define void @ldr_with_off_var(ptr %base, i32 %off) {300; CHECK-LABEL: ldr_with_off_var:301; CHECK: // %bb.0:302; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1303; CHECK-NEXT: sxtw x8, w1304; CHECK-NEXT: rdsvl x9, #1305; CHECK-NEXT: add w12, w1, #16306; CHECK-NEXT: madd x8, x9, x8, x0307; CHECK-NEXT: ldr za[w12, 0], [x8]308; CHECK-NEXT: ret309 call void @llvm.aarch64.sme.ldr(i32 16, ptr %base, i32 %off)310 ret void;311}312 313define void @ldr_with_off_15imm(ptr %base) {314; CHECK-LABEL: ldr_with_off_15imm:315; CHECK: // %bb.0:316; CHECK-NEXT: mov w12, #16 // =0x10317; CHECK-NEXT: ldr za[w12, 15], [x0, #15, mul vl]318; CHECK-NEXT: ret319 call void @llvm.aarch64.sme.ldr(i32 16, ptr %base, i32 15)320 ret void;321}322 323define void @ldr_with_off_16imm(ptr %base) {324; CHECK-LABEL: ldr_with_off_16imm:325; CHECK: // %bb.0:326; CHECK-NEXT: rdsvl x8, #1327; CHECK-NEXT: mov w12, #32 // =0x20328; CHECK-NEXT: add x8, x0, x8, lsl #4329; CHECK-NEXT: ldr za[w12, 0], [x8]330; CHECK-NEXT: ret331 call void @llvm.aarch64.sme.ldr(i32 16, ptr %base, i32 16)332 ret void;333}334 335define void @ldr_with_off_many_imm(i32 %tile_slice, ptr %ptr) {336; CHECK-LABEL: ldr_with_off_many_imm:337; CHECK: // %bb.0: // %entry338; CHECK-NEXT: mov w12, w0339; CHECK-NEXT: ldr za[w12, 1], [x1, #1, mul vl]340; CHECK-NEXT: ldr za[w12, 2], [x1, #2, mul vl]341; CHECK-NEXT: ldr za[w12, 3], [x1, #3, mul vl]342; CHECK-NEXT: ldr za[w12, 4], [x1, #4, mul vl]343; CHECK-NEXT: ret344entry:345 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 1)346 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 2)347 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 3)348 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 4)349 ret void350}351 352define void @ldr_with_off_many_imm_15_18(i32 %tile_slice, ptr %ptr) {353; CHECK-LABEL: ldr_with_off_many_imm_15_18:354; CHECK: // %bb.0: // %entry355; CHECK-NEXT: rdsvl x8, #1356; CHECK-NEXT: mov w12, w0357; CHECK-NEXT: add x8, x1, x8, lsl #4358; CHECK-NEXT: ldr za[w12, 15], [x1, #15, mul vl]359; CHECK-NEXT: add w12, w0, #16360; CHECK-NEXT: ldr za[w12, 0], [x8]361; CHECK-NEXT: ldr za[w12, 1], [x8, #1, mul vl]362; CHECK-NEXT: ldr za[w12, 2], [x8, #2, mul vl]363; CHECK-NEXT: ret364entry:365 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 15)366 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 16)367 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 17)368 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 18)369 ret void370}371 372define void @ldr_with_off_many_imm_16_19(i32 %tile_slice, ptr %ptr) {373; CHECK-LABEL: ldr_with_off_many_imm_16_19:374; CHECK: // %bb.0: // %entry375; CHECK-NEXT: rdsvl x8, #1376; CHECK-NEXT: add w12, w0, #16377; CHECK-NEXT: add x8, x1, x8, lsl #4378; CHECK-NEXT: ldr za[w12, 0], [x8]379; CHECK-NEXT: ldr za[w12, 1], [x8, #1, mul vl]380; CHECK-NEXT: ldr za[w12, 2], [x8, #2, mul vl]381; CHECK-NEXT: ldr za[w12, 3], [x8, #3, mul vl]382; CHECK-NEXT: ret383entry:384 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 16)385 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 17)386 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 18)387 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 19)388 ret void389}390 391define void @ldr_with_off_many_imm_31_34(i32 %tile_slice, ptr %ptr) {392; CHECK-LABEL: ldr_with_off_many_imm_31_34:393; CHECK: // %bb.0: // %entry394; CHECK-NEXT: rdsvl x8, #1395; CHECK-NEXT: add w12, w0, #16396; CHECK-NEXT: add x9, x1, x8, lsl #4397; CHECK-NEXT: add x8, x1, x8, lsl #5398; CHECK-NEXT: ldr za[w12, 15], [x9, #15, mul vl]399; CHECK-NEXT: add w12, w0, #32400; CHECK-NEXT: ldr za[w12, 0], [x8]401; CHECK-NEXT: ldr za[w12, 1], [x8, #1, mul vl]402; CHECK-NEXT: ldr za[w12, 2], [x8, #2, mul vl]403; CHECK-NEXT: ret404entry:405 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 31)406 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 32)407 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 33)408 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 34)409 ret void410}411 412define void @ldr_with_off_many_imm_32_35(i32 %tile_slice, ptr %ptr, i64 %vnum) {413; CHECK-LABEL: ldr_with_off_many_imm_32_35:414; CHECK: // %bb.0: // %entry415; CHECK-NEXT: rdsvl x8, #1416; CHECK-NEXT: add w12, w0, #32417; CHECK-NEXT: add x8, x1, x8, lsl #5418; CHECK-NEXT: ldr za[w12, 0], [x8]419; CHECK-NEXT: ldr za[w12, 1], [x8, #1, mul vl]420; CHECK-NEXT: ldr za[w12, 2], [x8, #2, mul vl]421; CHECK-NEXT: ldr za[w12, 3], [x8, #3, mul vl]422; CHECK-NEXT: ret423entry:424 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 32)425 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 33)426 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 34)427 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 35)428 ret void429}430 431define void @ldr_with_off_many_var(i32 %tile_slice, ptr %ptr, i64 %vnum) {432; CHECK-LABEL: ldr_with_off_many_var:433; CHECK: // %bb.0: // %entry434; CHECK-NEXT: sxtw x8, w2435; CHECK-NEXT: rdsvl x9, #1436; CHECK-NEXT: add w12, w0, w2437; CHECK-NEXT: madd x8, x9, x8, x1438; CHECK-NEXT: ldr za[w12, 0], [x8]439; CHECK-NEXT: ldr za[w12, 1], [x8, #1, mul vl]440; CHECK-NEXT: ldr za[w12, 2], [x8, #2, mul vl]441; CHECK-NEXT: ldr za[w12, 3], [x8, #3, mul vl]442; CHECK-NEXT: ret443entry:444 %0 = trunc i64 %vnum to i32445 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 %0)446 %1 = add i32 %0, 1447 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 %1)448 %2 = add i32 %0, 2449 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 %2)450 %3 = add i32 %0, 3451 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 %3)452 ret void453}454 455define void @ldr_with_off_many_var_high(i32 %tile_slice, ptr %ptr, i64 %vnum) {456; CHECK-LABEL: ldr_with_off_many_var_high:457; CHECK: // %bb.0: // %entry458; CHECK-NEXT: add w8, w2, #32459; CHECK-NEXT: rdsvl x10, #1460; CHECK-NEXT: sxtw x9, w8461; CHECK-NEXT: add w12, w0, w8462; CHECK-NEXT: madd x9, x10, x9, x1463; CHECK-NEXT: ldr za[w12, 1], [x9, #1, mul vl]464; CHECK-NEXT: ldr za[w12, 2], [x9, #2, mul vl]465; CHECK-NEXT: ldr za[w12, 3], [x9, #3, mul vl]466; CHECK-NEXT: ldr za[w12, 4], [x9, #4, mul vl]467; CHECK-NEXT: ret468entry:469 %0 = trunc i64 %vnum to i32470 %1 = add i32 %0, 33471 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 %1)472 %2 = add i32 %0, 34473 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 %2)474 %3 = add i32 %0, 35475 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 %3)476 %4 = add i32 %0, 36477 tail call void @llvm.aarch64.sme.ldr(i32 %tile_slice, ptr %ptr, i32 %4)478 ret void479}480 481; Ensure that the tile offset is sunk, given that this is likely to be an 'add'482; that's decomposed into a base + offset in ISel.483define void @test_ld1_sink_tile0_offset_operand(<vscale x 4 x i1> %pg, ptr %src, i32 %base, i32 %N) {484; CHECK-LABEL: test_ld1_sink_tile0_offset_operand:485; CHECK: // %bb.0: // %entry486; CHECK-NEXT: mov w12, w1487; CHECK-NEXT: .LBB24_1: // %for.body488; CHECK-NEXT: // =>This Inner Loop Header: Depth=1489; CHECK-NEXT: ld1w {za0h.s[w12, 0]}, p0/z, [x0]490; CHECK-NEXT: subs w2, w2, #1491; CHECK-NEXT: ld1w {za0h.s[w12, 1]}, p0/z, [x0]492; CHECK-NEXT: ld1w {za0h.s[w12, 2]}, p0/z, [x0]493; CHECK-NEXT: b.ne .LBB24_1494; CHECK-NEXT: // %bb.2: // %exit495; CHECK-NEXT: ret496entry:497 %add1 = add i32 %base, 1498 %add2 = add i32 %base, 2499 br label %for.body500 501for.body:502 %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]503 call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> %pg, ptr %src, i32 0, i32 %base)504 call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> %pg, ptr %src, i32 0, i32 %add1)505 call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> %pg, ptr %src, i32 0, i32 %add2)506 %inc = add nuw nsw i32 %i, 1507 %exitcond.not = icmp eq i32 %inc, %N508 br i1 %exitcond.not, label %exit, label %for.body509 510exit:511 ret void512}513 514 515declare void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1>, ptr, i32, i32)516declare void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1>, ptr, i32, i32)517declare void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1>, ptr, i32, i32)518declare void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1>, ptr, i32, i32)519declare void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1>, ptr, i32, i32)520declare void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1>, ptr, i32, i32)521declare void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1>, ptr, i32, i32)522declare void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1>, ptr, i32, i32)523declare void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1>, ptr, i32, i32)524declare void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1>, ptr, i32, i32)525 526declare void @llvm.aarch64.sme.ldr(i32, ptr, i32)527declare i64 @llvm.vscale.i64()528