brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -force-streaming -enable-subreg-liveness -verify-machineinstrs < %s | FileCheck %s3 4target triple="aarch64-linux-gnu"5 6 7; == Multi, multi (unsigned) ==8 9define void @udot_multi_za32_u16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3) #0 {10; CHECK-LABEL: udot_multi_za32_u16_vg1x2:11; CHECK:       // %bb.0:12; CHECK-NEXT:    mov z5.d, z4.d13; CHECK-NEXT:    mov z7.d, z2.d14; CHECK-NEXT:    mov w8, w015; CHECK-NEXT:    mov z4.d, z3.d16; CHECK-NEXT:    mov z6.d, z1.d17; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z6.h, z7.h }, { z4.h, z5.h }18; CHECK-NEXT:    udot za.s[w8, 7, vgx2], { z6.h, z7.h }, { z4.h, z5.h }19; CHECK-NEXT:    ret20  call void @llvm.aarch64.sme.udot.za32.vg1x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3)21  %slice2 = add i32 %slice, 722  call void @llvm.aarch64.sme.udot.za32.vg1x2.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3)23  ret void24}25 26define void @udot_multi_za32_u16_vg1x2_tuple(i64 %stride, ptr %ptr) #1 {27; CHECK-LABEL: udot_multi_za32_u16_vg1x2_tuple:28; CHECK:       // %bb.0: // %entry29; CHECK-NEXT:    ptrue pn8.b30; CHECK-NEXT:    mov w8, wzr31; CHECK-NEXT:    ld1b { z16.b, z24.b }, pn8/z, [x1]32; CHECK-NEXT:    ld1b { z17.b, z25.b }, pn8/z, [x1, x0]33; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z16.b, z17.b }, { z24.b, z25.b }34; CHECK-NEXT:    ret35entry:36  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()37  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)38  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 039  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 140  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride41  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)42  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 043  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 144  call void @llvm.aarch64.sme.udot.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6)45  ret void46}47 48define void @udot_multi_za32_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,49; CHECK-LABEL: udot_multi_za32_u16_vg1x4:50; CHECK:       // %bb.0:51; CHECK-NEXT:    mov z26.d, z7.d52; CHECK-NEXT:    mov z25.d, z6.d53; CHECK-NEXT:    ldr z27, [x1]54; CHECK-NEXT:    mov z7.d, z4.d55; CHECK-NEXT:    mov z24.d, z5.d56; CHECK-NEXT:    mov w8, w057; CHECK-NEXT:    mov z6.d, z3.d58; CHECK-NEXT:    mov z5.d, z2.d59; CHECK-NEXT:    mov z4.d, z1.d60; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z4.h - z7.h }, { z24.h - z27.h }61; CHECK-NEXT:    udot za.s[w8, 7, vgx4], { z4.h - z7.h }, { z24.h - z27.h }62; CHECK-NEXT:    ret63                                        <vscale x 8 x i16> %zn4, <vscale x 8 x i16> %zn5, <vscale x 8 x i16> %zn6, <vscale x 8 x i16> %zn7) #0 {64  call void @llvm.aarch64.sme.udot.za32.vg1x4.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,65                                                      <vscale x 8 x i16> %zn4, <vscale x 8 x i16> %zn5, <vscale x 8 x i16> %zn6, <vscale x 8 x i16> %zn7)66  %slice2 = add i32 %slice, 767  call void @llvm.aarch64.sme.udot.za32.vg1x4.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,68                                                      <vscale x 8 x i16> %zn4, <vscale x 8 x i16> %zn5, <vscale x 8 x i16> %zn6, <vscale x 8 x i16> %zn7)69  ret void70}71 72define void @udot_multi_za32_u16_vg1x4_tuple(i64 %stride, ptr %ptr) #1 {73; CHECK-LABEL: udot_multi_za32_u16_vg1x4_tuple:74; CHECK:       // %bb.0: // %entry75; CHECK-NEXT:    lsl x9, x0, #176; CHECK-NEXT:    ptrue pn8.b77; CHECK-NEXT:    mov w8, wzr78; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x1]79; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x1, x0]80; CHECK-NEXT:    add x10, x9, x081; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x1, x9]82; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x1, x10]83; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z16.b - z19.b }, { z20.b - z23.b }84; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z24.b - z27.b }, { z28.b - z31.b }85; CHECK-NEXT:    ret86entry:87  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()88  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)89  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 090  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 191  %4 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 292  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 393  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride94  %6 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)95  %7 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 096  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 197  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 298  %10 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 399  %mul3 = shl i64 %stride, 1100  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3101  %11 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)102  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 0103  %13 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 1104  %14 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 2105  %15 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 3106  %mul5 = mul i64 %stride, 3107  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5108  %16 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)109  %17 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 0110  %18 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 1111  %19 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 2112  %20 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 3113  call void @llvm.aarch64.sme.udot.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %7, <vscale x 16 x i8> %12, <vscale x 16 x i8> %17,114                                                      <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18)115  call void @llvm.aarch64.sme.udot.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19,116                                                      <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20)117  ret void118}119 120define void @udot_multi_za32_u8_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3) #0 {121; CHECK-LABEL: udot_multi_za32_u8_vg1x2:122; CHECK:       // %bb.0:123; CHECK-NEXT:    mov z5.d, z4.d124; CHECK-NEXT:    mov z7.d, z2.d125; CHECK-NEXT:    mov w8, w0126; CHECK-NEXT:    mov z4.d, z3.d127; CHECK-NEXT:    mov z6.d, z1.d128; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z6.b, z7.b }, { z4.b, z5.b }129; CHECK-NEXT:    udot za.s[w8, 7, vgx2], { z6.b, z7.b }, { z4.b, z5.b }130; CHECK-NEXT:    ret131  call void @llvm.aarch64.sme.udot.za32.vg1x2.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3)132  %slice2 = add i32 %slice, 7133  call void @llvm.aarch64.sme.udot.za32.vg1x2.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3)134  ret void135}136 137define void @udot_multi_za32_u8_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,138; CHECK-LABEL: udot_multi_za32_u8_vg1x4:139; CHECK:       // %bb.0:140; CHECK-NEXT:    mov z26.d, z7.d141; CHECK-NEXT:    mov z25.d, z6.d142; CHECK-NEXT:    ldr z27, [x1]143; CHECK-NEXT:    mov z7.d, z4.d144; CHECK-NEXT:    mov z24.d, z5.d145; CHECK-NEXT:    mov w8, w0146; CHECK-NEXT:    mov z6.d, z3.d147; CHECK-NEXT:    mov z5.d, z2.d148; CHECK-NEXT:    mov z4.d, z1.d149; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z4.b - z7.b }, { z24.b - z27.b }150; CHECK-NEXT:    udot za.s[w8, 7, vgx4], { z4.b - z7.b }, { z24.b - z27.b }151; CHECK-NEXT:    ret152                                      <vscale x 16 x i8> %zn4, <vscale x 16 x i8> %zn5, <vscale x 16 x i8> %zn6, <vscale x 16 x i8> %zn7) #0 {153  call void @llvm.aarch64.sme.udot.za32.vg1x4.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,154                                                      <vscale x 16 x i8> %zn4, <vscale x 16 x i8> %zn5, <vscale x 16 x i8> %zn6, <vscale x 16 x i8> %zn7)155  %slice2 = add i32 %slice, 7156  call void @llvm.aarch64.sme.udot.za32.vg1x4.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,157                                                      <vscale x 16 x i8> %zn4, <vscale x 16 x i8> %zn5, <vscale x 16 x i8> %zn6, <vscale x 16 x i8> %zn7)158  ret void159}160 161define void @udot_multi_za64_u16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3) #1 {162; CHECK-LABEL: udot_multi_za64_u16_vg1x2:163; CHECK:       // %bb.0:164; CHECK-NEXT:    mov z5.d, z4.d165; CHECK-NEXT:    mov z7.d, z2.d166; CHECK-NEXT:    mov w8, w0167; CHECK-NEXT:    mov z4.d, z3.d168; CHECK-NEXT:    mov z6.d, z1.d169; CHECK-NEXT:    udot za.d[w8, 0, vgx2], { z6.h, z7.h }, { z4.h, z5.h }170; CHECK-NEXT:    udot za.d[w8, 7, vgx2], { z6.h, z7.h }, { z4.h, z5.h }171; CHECK-NEXT:    ret172  call void @llvm.aarch64.sme.udot.za64.vg1x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3)173  %slice2 = add i32 %slice, 7174  call void @llvm.aarch64.sme.udot.za64.vg1x2.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3)175  ret void176}177 178define void @udot_multi_za64_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,179; CHECK-LABEL: udot_multi_za64_u16_vg1x4:180; CHECK:       // %bb.0:181; CHECK-NEXT:    mov z26.d, z7.d182; CHECK-NEXT:    mov z25.d, z6.d183; CHECK-NEXT:    ldr z27, [x1]184; CHECK-NEXT:    mov z7.d, z4.d185; CHECK-NEXT:    mov z24.d, z5.d186; CHECK-NEXT:    mov w8, w0187; CHECK-NEXT:    mov z6.d, z3.d188; CHECK-NEXT:    mov z5.d, z2.d189; CHECK-NEXT:    mov z4.d, z1.d190; CHECK-NEXT:    udot za.d[w8, 0, vgx4], { z4.h - z7.h }, { z24.h - z27.h }191; CHECK-NEXT:    udot za.d[w8, 7, vgx4], { z4.h - z7.h }, { z24.h - z27.h }192; CHECK-NEXT:    ret193                                       <vscale x 8 x i16> %zn4, <vscale x 8 x i16> %zn5, <vscale x 8 x i16> %zn6, <vscale x 8 x i16> %zn7) #1 {194  call void @llvm.aarch64.sme.udot.za64.vg1x4.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,195                                                      <vscale x 8 x i16> %zn4, <vscale x 8 x i16> %zn5, <vscale x 8 x i16> %zn6, <vscale x 8 x i16> %zn7)196  %slice2 = add i32 %slice, 7197  call void @llvm.aarch64.sme.udot.za64.vg1x4.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,198                                                      <vscale x 8 x i16> %zn4, <vscale x 8 x i16> %zn5, <vscale x 8 x i16> %zn6, <vscale x 8 x i16> %zn7)199  ret void200}201 202define void @usdot_multi_za32_u8_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3) #0 {203; CHECK-LABEL: usdot_multi_za32_u8_vg1x2:204; CHECK:       // %bb.0:205; CHECK-NEXT:    mov z5.d, z4.d206; CHECK-NEXT:    mov z7.d, z2.d207; CHECK-NEXT:    mov w8, w0208; CHECK-NEXT:    mov z4.d, z3.d209; CHECK-NEXT:    mov z6.d, z1.d210; CHECK-NEXT:    usdot za.s[w8, 0, vgx2], { z6.b, z7.b }, { z4.b, z5.b }211; CHECK-NEXT:    usdot za.s[w8, 7, vgx2], { z6.b, z7.b }, { z4.b, z5.b }212; CHECK-NEXT:    ret213  call void @llvm.aarch64.sme.usdot.za32.vg1x2.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3)214  %slice2 = add i32 %slice, 7215  call void @llvm.aarch64.sme.usdot.za32.vg1x2.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3)216  ret void217}218 219define void @usdot_multi_za32_u16_vg1x2_tuple(i64 %stride, ptr %ptr) #1 {220; CHECK-LABEL: usdot_multi_za32_u16_vg1x2_tuple:221; CHECK:       // %bb.0: // %entry222; CHECK-NEXT:    ptrue pn8.b223; CHECK-NEXT:    mov w8, wzr224; CHECK-NEXT:    ld1b { z16.b, z24.b }, pn8/z, [x1]225; CHECK-NEXT:    ld1b { z17.b, z25.b }, pn8/z, [x1, x0]226; CHECK-NEXT:    usdot za.s[w8, 0, vgx2], { z16.b, z17.b }, { z24.b, z25.b }227; CHECK-NEXT:    ret228entry:229  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()230  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)231  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 0232  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 1233  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride234  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)235  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 0236  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 1237  call void @llvm.aarch64.sme.usdot.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6)238  ret void239}240 241define void @usdot_multi_za32_u8_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,242; CHECK-LABEL: usdot_multi_za32_u8_vg1x4:243; CHECK:       // %bb.0:244; CHECK-NEXT:    mov z26.d, z7.d245; CHECK-NEXT:    mov z25.d, z6.d246; CHECK-NEXT:    ldr z27, [x1]247; CHECK-NEXT:    mov z7.d, z4.d248; CHECK-NEXT:    mov z24.d, z5.d249; CHECK-NEXT:    mov w8, w0250; CHECK-NEXT:    mov z6.d, z3.d251; CHECK-NEXT:    mov z5.d, z2.d252; CHECK-NEXT:    mov z4.d, z1.d253; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z4.b - z7.b }, { z24.b - z27.b }254; CHECK-NEXT:    usdot za.s[w8, 7, vgx4], { z4.b - z7.b }, { z24.b - z27.b }255; CHECK-NEXT:    ret256                                      <vscale x 16 x i8> %zn4, <vscale x 16 x i8> %zn5, <vscale x 16 x i8> %zn6, <vscale x 16 x i8> %zn7) #0 {257  call void @llvm.aarch64.sme.usdot.za32.vg1x4.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,258                                                      <vscale x 16 x i8> %zn4, <vscale x 16 x i8> %zn5, <vscale x 16 x i8> %zn6, <vscale x 16 x i8> %zn7)259  %slice2 = add i32 %slice, 7260  call void @llvm.aarch64.sme.usdot.za32.vg1x4.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,261                                                      <vscale x 16 x i8> %zn4, <vscale x 16 x i8> %zn5, <vscale x 16 x i8> %zn6, <vscale x 16 x i8> %zn7)262  ret void263}264 265define void @usdot_multi_za32_u16_vg1x4_tuple(i64 %stride, ptr %ptr) #1 {266; CHECK-LABEL: usdot_multi_za32_u16_vg1x4_tuple:267; CHECK:       // %bb.0: // %entry268; CHECK-NEXT:    lsl x9, x0, #1269; CHECK-NEXT:    ptrue pn8.b270; CHECK-NEXT:    mov w8, wzr271; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x1]272; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x1, x0]273; CHECK-NEXT:    add x10, x9, x0274; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x1, x9]275; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x1, x10]276; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z16.b - z19.b }, { z20.b - z23.b }277; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z24.b - z27.b }, { z28.b - z31.b }278; CHECK-NEXT:    ret279entry:280  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()281  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)282  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 0283  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 1284  %4 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 2285  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 3286  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride287  %6 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)288  %7 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 0289  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 1290  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 2291  %10 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 3292  %mul3 = shl i64 %stride, 1293  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3294  %11 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)295  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 0296  %13 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 1297  %14 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 2298  %15 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 3299  %mul5 = mul i64 %stride, 3300  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5301  %16 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)302  %17 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 0303  %18 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 1304  %19 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 2305  %20 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 3306  call void @llvm.aarch64.sme.usdot.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %7, <vscale x 16 x i8> %12, <vscale x 16 x i8> %17,307                                                      <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18)308  call void @llvm.aarch64.sme.usdot.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19,309                                                      <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20)310  ret void311}312 313; == Multi, multi (signed) ==314 315define void @sdot_multi_za32_u16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3) #0 {316; CHECK-LABEL: sdot_multi_za32_u16_vg1x2:317; CHECK:       // %bb.0:318; CHECK-NEXT:    mov z5.d, z4.d319; CHECK-NEXT:    mov z7.d, z2.d320; CHECK-NEXT:    mov w8, w0321; CHECK-NEXT:    mov z4.d, z3.d322; CHECK-NEXT:    mov z6.d, z1.d323; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z6.h, z7.h }, { z4.h, z5.h }324; CHECK-NEXT:    sdot za.s[w8, 7, vgx2], { z6.h, z7.h }, { z4.h, z5.h }325; CHECK-NEXT:    ret326  call void @llvm.aarch64.sme.sdot.za32.vg1x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3)327  %slice2 = add i32 %slice, 7328  call void @llvm.aarch64.sme.sdot.za32.vg1x2.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3)329  ret void330}331 332define void @sdot_multi_za32_u16_vg1x2_tuple(i64 %stride, ptr %ptr) #0 {333; CHECK-LABEL: sdot_multi_za32_u16_vg1x2_tuple:334; CHECK:       // %bb.0: // %entry335; CHECK-NEXT:    ptrue pn8.b336; CHECK-NEXT:    mov w8, wzr337; CHECK-NEXT:    ld1b { z16.b, z24.b }, pn8/z, [x1]338; CHECK-NEXT:    ld1b { z17.b, z25.b }, pn8/z, [x1, x0]339; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z16.b, z17.b }, { z24.b, z25.b }340; CHECK-NEXT:    ret341entry:342  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()343  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)344  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 0345  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 1346  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride347  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)348  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 0349  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 1350  call void @llvm.aarch64.sme.sdot.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6)351  ret void352}353 354define void @sdot_multi_za32_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,355; CHECK-LABEL: sdot_multi_za32_u16_vg1x4:356; CHECK:       // %bb.0:357; CHECK-NEXT:    mov z26.d, z7.d358; CHECK-NEXT:    mov z25.d, z6.d359; CHECK-NEXT:    ldr z27, [x1]360; CHECK-NEXT:    mov z7.d, z4.d361; CHECK-NEXT:    mov z24.d, z5.d362; CHECK-NEXT:    mov w8, w0363; CHECK-NEXT:    mov z6.d, z3.d364; CHECK-NEXT:    mov z5.d, z2.d365; CHECK-NEXT:    mov z4.d, z1.d366; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z4.h - z7.h }, { z24.h - z27.h }367; CHECK-NEXT:    sdot za.s[w8, 7, vgx4], { z4.h - z7.h }, { z24.h - z27.h }368; CHECK-NEXT:    ret369                                        <vscale x 8 x i16> %zn4, <vscale x 8 x i16> %zn5, <vscale x 8 x i16> %zn6, <vscale x 8 x i16> %zn7) #0 {370  call void @llvm.aarch64.sme.sdot.za32.vg1x4.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,371                                                      <vscale x 8 x i16> %zn4, <vscale x 8 x i16> %zn5, <vscale x 8 x i16> %zn6, <vscale x 8 x i16> %zn7)372  %slice2 = add i32 %slice, 7373  call void @llvm.aarch64.sme.sdot.za32.vg1x4.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,374                                                      <vscale x 8 x i16> %zn4, <vscale x 8 x i16> %zn5, <vscale x 8 x i16> %zn6, <vscale x 8 x i16> %zn7)375  ret void376}377 378define void @sdot_multi_za32_u16_vg1x4_tuple(i64 %stride, ptr %ptr) #0 {379; CHECK-LABEL: sdot_multi_za32_u16_vg1x4_tuple:380; CHECK:       // %bb.0: // %entry381; CHECK-NEXT:    lsl x9, x0, #1382; CHECK-NEXT:    ptrue pn8.b383; CHECK-NEXT:    mov w8, wzr384; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x1]385; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x1, x0]386; CHECK-NEXT:    add x10, x9, x0387; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x1, x9]388; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x1, x10]389; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z16.b - z19.b }, { z20.b - z23.b }390; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z24.b - z27.b }, { z28.b - z31.b }391; CHECK-NEXT:    ret392entry:393  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()394  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)395  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 0396  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 1397  %4 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 2398  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 3399  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride400  %6 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)401  %7 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 0402  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 1403  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 2404  %10 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 3405  %mul3 = shl i64 %stride, 1406  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3407  %11 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)408  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 0409  %13 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 1410  %14 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 2411  %15 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 3412  %mul5 = mul i64 %stride, 3413  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5414  %16 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)415  %17 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 0416  %18 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 1417  %19 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 2418  %20 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 3419  call void @llvm.aarch64.sme.sdot.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %7, <vscale x 16 x i8> %12, <vscale x 16 x i8> %17,420                                                      <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18)421  call void @llvm.aarch64.sme.sdot.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19,422                                                      <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20)423  ret void424}425 426define void @sdot_multi_za32_u8_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3) #0 {427; CHECK-LABEL: sdot_multi_za32_u8_vg1x2:428; CHECK:       // %bb.0:429; CHECK-NEXT:    mov z5.d, z4.d430; CHECK-NEXT:    mov z7.d, z2.d431; CHECK-NEXT:    mov w8, w0432; CHECK-NEXT:    mov z4.d, z3.d433; CHECK-NEXT:    mov z6.d, z1.d434; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z6.b, z7.b }, { z4.b, z5.b }435; CHECK-NEXT:    sdot za.s[w8, 7, vgx2], { z6.b, z7.b }, { z4.b, z5.b }436; CHECK-NEXT:    ret437  call void @llvm.aarch64.sme.sdot.za32.vg1x2.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3)438  %slice2 = add i32 %slice, 7439  call void @llvm.aarch64.sme.sdot.za32.vg1x2.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3)440  ret void441}442 443define void @sdot_multi_za32_u8_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,444; CHECK-LABEL: sdot_multi_za32_u8_vg1x4:445; CHECK:       // %bb.0:446; CHECK-NEXT:    mov z26.d, z7.d447; CHECK-NEXT:    mov z25.d, z6.d448; CHECK-NEXT:    ldr z27, [x1]449; CHECK-NEXT:    mov z7.d, z4.d450; CHECK-NEXT:    mov z24.d, z5.d451; CHECK-NEXT:    mov w8, w0452; CHECK-NEXT:    mov z6.d, z3.d453; CHECK-NEXT:    mov z5.d, z2.d454; CHECK-NEXT:    mov z4.d, z1.d455; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z4.b - z7.b }, { z24.b - z27.b }456; CHECK-NEXT:    sdot za.s[w8, 7, vgx4], { z4.b - z7.b }, { z24.b - z27.b }457; CHECK-NEXT:    ret458                                      <vscale x 16 x i8> %zn4, <vscale x 16 x i8> %zn5, <vscale x 16 x i8> %zn6, <vscale x 16 x i8> %zn7) #0 {459  call void @llvm.aarch64.sme.sdot.za32.vg1x4.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,460                                                      <vscale x 16 x i8> %zn4, <vscale x 16 x i8> %zn5, <vscale x 16 x i8> %zn6, <vscale x 16 x i8> %zn7)461  %slice2 = add i32 %slice, 7462  call void @llvm.aarch64.sme.sdot.za32.vg1x4.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,463                                                      <vscale x 16 x i8> %zn4, <vscale x 16 x i8> %zn5, <vscale x 16 x i8> %zn6, <vscale x 16 x i8> %zn7)464  ret void465}466 467define void @sdot_multi_za64_u16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3) #1 {468; CHECK-LABEL: sdot_multi_za64_u16_vg1x2:469; CHECK:       // %bb.0:470; CHECK-NEXT:    mov z5.d, z4.d471; CHECK-NEXT:    mov z7.d, z2.d472; CHECK-NEXT:    mov w8, w0473; CHECK-NEXT:    mov z4.d, z3.d474; CHECK-NEXT:    mov z6.d, z1.d475; CHECK-NEXT:    sdot za.d[w8, 0, vgx2], { z6.h, z7.h }, { z4.h, z5.h }476; CHECK-NEXT:    sdot za.d[w8, 7, vgx2], { z6.h, z7.h }, { z4.h, z5.h }477; CHECK-NEXT:    ret478  call void @llvm.aarch64.sme.sdot.za64.vg1x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3)479  %slice2 = add i32 %slice, 7480  call void @llvm.aarch64.sme.sdot.za64.vg1x2.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3)481  ret void482}483 484define void @sdot_multi_za64_u16_vg1x2_tuple(i64 %stride, ptr %ptr) #1 {485; CHECK-LABEL: sdot_multi_za64_u16_vg1x2_tuple:486; CHECK:       // %bb.0: // %entry487; CHECK-NEXT:    ptrue pn8.b488; CHECK-NEXT:    add x9, x1, x0489; CHECK-NEXT:    mov w8, wzr490; CHECK-NEXT:    ld1h { z16.h, z24.h }, pn8/z, [x1]491; CHECK-NEXT:    ld1h { z17.h, z25.h }, pn8/z, [x9]492; CHECK-NEXT:    sdot za.d[w8, 0, vgx2], { z16.h, z17.h }, { z24.h, z25.h }493; CHECK-NEXT:    ret494entry:495  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()496  %1 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") %0, ptr %ptr)497  %2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 0498  %3 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 1499  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride500  %4 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx2)501  %5 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %4, 0502  %6 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %4, 1503  call void @llvm.aarch64.sme.sdot.za64.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %2, <vscale x 8 x i16> %5, <vscale x 8 x i16> %3, <vscale x 8 x i16> %6)504  ret void505}506 507define void @sdot_multi_za64_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,508; CHECK-LABEL: sdot_multi_za64_u16_vg1x4:509; CHECK:       // %bb.0:510; CHECK-NEXT:    mov z26.d, z7.d511; CHECK-NEXT:    mov z25.d, z6.d512; CHECK-NEXT:    ldr z27, [x1]513; CHECK-NEXT:    mov z7.d, z4.d514; CHECK-NEXT:    mov z24.d, z5.d515; CHECK-NEXT:    mov w8, w0516; CHECK-NEXT:    mov z6.d, z3.d517; CHECK-NEXT:    mov z5.d, z2.d518; CHECK-NEXT:    mov z4.d, z1.d519; CHECK-NEXT:    sdot za.d[w8, 0, vgx4], { z4.h - z7.h }, { z24.h - z27.h }520; CHECK-NEXT:    sdot za.d[w8, 7, vgx4], { z4.h - z7.h }, { z24.h - z27.h }521; CHECK-NEXT:    ret522                                       <vscale x 8 x i16> %zn4, <vscale x 8 x i16> %zn5, <vscale x 8 x i16> %zn6, <vscale x 8 x i16> %zn7) #1 {523  call void @llvm.aarch64.sme.sdot.za64.vg1x4.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,524                                                      <vscale x 8 x i16> %zn4, <vscale x 8 x i16> %zn5, <vscale x 8 x i16> %zn6, <vscale x 8 x i16> %zn7)525  %slice2 = add i32 %slice, 7526  call void @llvm.aarch64.sme.sdot.za64.vg1x4.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,527                                                      <vscale x 8 x i16> %zn4, <vscale x 8 x i16> %zn5, <vscale x 8 x i16> %zn6, <vscale x 8 x i16> %zn7)528  ret void529}530 531define void @sdot_multi_za64_u16_vg1x4_tuple(i64 %stride, ptr %ptr) #1 {532; CHECK-LABEL: sdot_multi_za64_u16_vg1x4_tuple:533; CHECK:       // %bb.0: // %entry534; CHECK-NEXT:    add x9, x0, x0, lsl #1535; CHECK-NEXT:    ptrue pn8.b536; CHECK-NEXT:    add x10, x1, x0537; CHECK-NEXT:    ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x1]538; CHECK-NEXT:    ld1h { z17.h, z21.h, z25.h, z29.h }, pn8/z, [x10]539; CHECK-NEXT:    ld1h { z18.h, z22.h, z26.h, z30.h }, pn8/z, [x1, x0, lsl #1]540; CHECK-NEXT:    add x9, x1, x9541; CHECK-NEXT:    mov w8, wzr542; CHECK-NEXT:    ld1h { z19.h, z23.h, z27.h, z31.h }, pn8/z, [x9]543; CHECK-NEXT:    sdot za.d[w8, 0, vgx4], { z16.h - z19.h }, { z20.h - z23.h }544; CHECK-NEXT:    sdot za.d[w8, 0, vgx4], { z24.h - z27.h }, { z28.h - z31.h }545; CHECK-NEXT:    ret546entry:547  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()548  %1 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %ptr)549  %2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 0550  %3 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 1551  %4 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 2552  %5 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 3553  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride554  %6 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx2)555  %7 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 0556  %8 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 1557  %9 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 2558  %10 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 3559  %mul3 = shl i64 %stride, 1560  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3561  %11 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx4)562  %12 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %11, 0563  %13 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %11, 1564  %14 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %11, 2565  %15 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %11, 3566  %mul5 = mul i64 %stride, 3567  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5568  %16 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx6)569  %17 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %16, 0570  %18 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %16, 1571  %19 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %16, 2572  %20 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %16, 3573  call void @llvm.aarch64.sme.sdot.za64.vg1x4.nxv8i16(i32 0, <vscale x 8 x i16> %2, <vscale x 8 x i16> %7, <vscale x 8 x i16> %12, <vscale x 8 x i16> %17,574                                                      <vscale x 8 x i16> %3, <vscale x 8 x i16> %8, <vscale x 8 x i16> %13, <vscale x 8 x i16> %18)575  call void @llvm.aarch64.sme.sdot.za64.vg1x4.nxv8i16(i32 0, <vscale x 8 x i16> %4, <vscale x 8 x i16> %9, <vscale x 8 x i16> %14, <vscale x 8 x i16> %19,576                                                      <vscale x 8 x i16> %5, <vscale x 8 x i16> %10, <vscale x 8 x i16> %15, <vscale x 8 x i16> %20)577  ret void578}579 580; == Multi, single (unsigned) ==581 582define void @udot_single_za32_u16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2) #0 {583; CHECK-LABEL: udot_single_za32_u16_vg1x2:584; CHECK:       // %bb.0:585; CHECK-NEXT:    mov w8, w0586; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z1.h, z2.h }, z3.h587; CHECK-NEXT:    udot za.s[w8, 7, vgx2], { z1.h, z2.h }, z3.h588; CHECK-NEXT:    ret589  call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2)590  %slice2 = add i32 %slice, 7591  call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2)592  ret void593}594 595define void @udot_single_za32_u16_vg1x2_tuple(ptr %ptr, i64 %stride, <vscale x 8 x i16> %zn) #0 {596; CHECK-LABEL: udot_single_za32_u16_vg1x2_tuple:597; CHECK:       // %bb.0: // %entry598; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill599; CHECK-NEXT:    addvl sp, sp, #-3600; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill601; CHECK-NEXT:    ptrue pn8.b602; CHECK-NEXT:    add x9, x0, x1603; CHECK-NEXT:    str z10, [sp, #1, mul vl] // 16-byte Folded Spill604; CHECK-NEXT:    mov w8, wzr605; CHECK-NEXT:    str z9, [sp, #2, mul vl] // 16-byte Folded Spill606; CHECK-NEXT:    ld1h { z1.h, z9.h }, pn8/z, [x0]607; CHECK-NEXT:    ld1h { z2.h, z10.h }, pn8/z, [x9]608; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z1.h, z2.h }, z0.h609; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z9.h, z10.h }, z0.h610; CHECK-NEXT:    ldr z10, [sp, #1, mul vl] // 16-byte Folded Reload611; CHECK-NEXT:    ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload612; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload613; CHECK-NEXT:    addvl sp, sp, #3614; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload615; CHECK-NEXT:    ret616entry:617  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()618  %1 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") %0, ptr %ptr)619  %2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 0620  %3 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 1621  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride622  %4 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx2)623  %5 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %4, 0624  %6 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %4, 1625  call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %2, <vscale x 8 x i16> %5, <vscale x 8 x i16> %zn)626  call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %3, <vscale x 8 x i16> %6, <vscale x 8 x i16> %zn)627  ret void628}629 630define void @udot_single_za32_u16_vg1x2_x4load_x2tuple(ptr %ptr, i64 %stride, <vscale x 8 x i16> %zn) #0 {631; CHECK-LABEL: udot_single_za32_u16_vg1x2_x4load_x2tuple:632; CHECK:       // %bb.0: // %entry633; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill634; CHECK-NEXT:    addvl sp, sp, #-5635; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill636; CHECK-NEXT:    ptrue pn8.b637; CHECK-NEXT:    add x9, x0, x1638; CHECK-NEXT:    str z14, [sp, #1, mul vl] // 16-byte Folded Spill639; CHECK-NEXT:    mov w8, wzr640; CHECK-NEXT:    str z13, [sp, #2, mul vl] // 16-byte Folded Spill641; CHECK-NEXT:    str z10, [sp, #3, mul vl] // 16-byte Folded Spill642; CHECK-NEXT:    str z9, [sp, #4, mul vl] // 16-byte Folded Spill643; CHECK-NEXT:    ld1h { z1.h, z5.h, z9.h, z13.h }, pn8/z, [x0]644; CHECK-NEXT:    ld1h { z2.h, z6.h, z10.h, z14.h }, pn8/z, [x9]645; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z1.h, z2.h }, z0.h646; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z5.h, z6.h }, z0.h647; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z9.h, z10.h }, z0.h648; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z13.h, z14.h }, z0.h649; CHECK-NEXT:    ldr z14, [sp, #1, mul vl] // 16-byte Folded Reload650; CHECK-NEXT:    ldr z13, [sp, #2, mul vl] // 16-byte Folded Reload651; CHECK-NEXT:    ldr z10, [sp, #3, mul vl] // 16-byte Folded Reload652; CHECK-NEXT:    ldr z9, [sp, #4, mul vl] // 16-byte Folded Reload653; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload654; CHECK-NEXT:    addvl sp, sp, #5655; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload656; CHECK-NEXT:    ret657entry:658  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()659  %1 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %ptr)660  %2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 0661  %3 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 1662  %4 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 2663  %5 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 3664  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride665  %6 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx2)666  %7 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 0667  %8 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 1668  %9 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 2669  %10 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 3670  call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %2, <vscale x 8 x i16> %7, <vscale x 8 x i16> %zn)671  call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %3, <vscale x 8 x i16> %8, <vscale x 8 x i16> %zn)672  call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %4, <vscale x 8 x i16> %9, <vscale x 8 x i16> %zn)673  call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %5, <vscale x 8 x i16> %10, <vscale x 8 x i16> %zn)674  ret void675}676 677define void @udot_single_za32_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4) #0 {678; CHECK-LABEL: udot_single_za32_u16_vg1x4:679; CHECK:       // %bb.0:680; CHECK-NEXT:    mov w8, w0681; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z1.h - z4.h }, z5.h682; CHECK-NEXT:    udot za.s[w8, 7, vgx4], { z1.h - z4.h }, z5.h683; CHECK-NEXT:    ret684  call void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4)685  %slice2 = add i32 %slice, 7686  call void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4)687  ret void688}689 690define void @udot_single_za32_u16_vg1x4_tuple(ptr %ptr, i64 %stride, <vscale x 8 x i16> %zn) #0 {691; CHECK-LABEL: udot_single_za32_u16_vg1x4_tuple:692; CHECK:       // %bb.0: // %entry693; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill694; CHECK-NEXT:    addvl sp, sp, #-9695; CHECK-NEXT:    add x9, x1, x1, lsl #1696; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill697; CHECK-NEXT:    ptrue pn8.b698; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill699; CHECK-NEXT:    add x10, x0, x1700; CHECK-NEXT:    mov w8, wzr701; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill702; CHECK-NEXT:    add x9, x0, x9703; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill704; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill705; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill706; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill707; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill708; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill709; CHECK-NEXT:    ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0]710; CHECK-NEXT:    ld1h { z17.h, z21.h, z25.h, z29.h }, pn8/z, [x10]711; CHECK-NEXT:    ld1h { z18.h, z22.h, z26.h, z30.h }, pn8/z, [x0, x1, lsl #1]712; CHECK-NEXT:    ld1h { z19.h, z23.h, z27.h, z31.h }, pn8/z, [x9]713; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z16.h - z19.h }, z0.h714; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z20.h - z23.h }, z0.h715; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z24.h - z27.h }, z0.h716; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z28.h - z31.h }, z0.h717; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload718; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload719; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload720; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload721; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload722; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload723; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload724; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload725; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload726; CHECK-NEXT:    addvl sp, sp, #9727; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload728; CHECK-NEXT:    ret729entry:730  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()731  %1 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %ptr)732  %2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 0733  %3 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 1734  %4 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 2735  %5 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 3736  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride737  %6 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx2)738  %7 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 0739  %8 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 1740  %9 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 2741  %10 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 3742  %mul3 = shl i64 %stride, 1743  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3744  %11 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx4)745  %12 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %11, 0746  %13 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %11, 1747  %14 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %11, 2748  %15 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %11, 3749  %mul5 = mul i64 %stride, 3750  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5751  %16 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx6)752  %17 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %16, 0753  %18 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %16, 1754  %19 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %16, 2755  %20 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %16, 3756  call void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv8i16(i32 0, <vscale x 8 x i16> %2, <vscale x 8 x i16> %7, <vscale x 8 x i16> %12, <vscale x 8 x i16> %17, <vscale x 8 x i16> %zn)757  call void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv8i16(i32 0, <vscale x 8 x i16> %3, <vscale x 8 x i16> %8, <vscale x 8 x i16> %13, <vscale x 8 x i16> %18, <vscale x 8 x i16> %zn)758  call void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv8i16(i32 0, <vscale x 8 x i16> %4, <vscale x 8 x i16> %9, <vscale x 8 x i16> %14, <vscale x 8 x i16> %19, <vscale x 8 x i16> %zn)759  call void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv8i16(i32 0, <vscale x 8 x i16> %5, <vscale x 8 x i16> %10, <vscale x 8 x i16> %15, <vscale x 8 x i16> %20, <vscale x 8 x i16> %zn)760  ret void761}762 763define void @udot_single_za32_u8_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2) #0 {764; CHECK-LABEL: udot_single_za32_u8_vg1x2:765; CHECK:       // %bb.0:766; CHECK-NEXT:    mov w8, w0767; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z1.b, z2.b }, z3.b768; CHECK-NEXT:    udot za.s[w8, 7, vgx2], { z1.b, z2.b }, z3.b769; CHECK-NEXT:    ret770  call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2)771  %slice2 = add i32 %slice, 7772  call void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2)773  ret void774}775 776define void @udot_single_za32_u8_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4) #0 {777; CHECK-LABEL: udot_single_za32_u8_vg1x4:778; CHECK:       // %bb.0:779; CHECK-NEXT:    mov w8, w0780; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z1.b - z4.b }, z5.b781; CHECK-NEXT:    udot za.s[w8, 7, vgx4], { z1.b - z4.b }, z5.b782; CHECK-NEXT:    ret783  call void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4)784  %slice2 = add i32 %slice, 7785  call void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4)786  ret void787}788 789define void @udot_single_za64_u16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2) #1 {790; CHECK-LABEL: udot_single_za64_u16_vg1x2:791; CHECK:       // %bb.0:792; CHECK-NEXT:    mov w8, w0793; CHECK-NEXT:    udot za.d[w8, 0, vgx2], { z1.h, z2.h }, z3.h794; CHECK-NEXT:    udot za.d[w8, 7, vgx2], { z1.h, z2.h }, z3.h795; CHECK-NEXT:    ret796  call void @llvm.aarch64.sme.udot.single.za64.vg1x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2)797  %slice2 = add i32 %slice, 7798  call void @llvm.aarch64.sme.udot.single.za64.vg1x2.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2)799  ret void800}801 802define void @udot_single_za64_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4) #1 {803; CHECK-LABEL: udot_single_za64_u16_vg1x4:804; CHECK:       // %bb.0:805; CHECK-NEXT:    mov w8, w0806; CHECK-NEXT:    udot za.d[w8, 0, vgx4], { z1.h - z4.h }, z5.h807; CHECK-NEXT:    udot za.d[w8, 7, vgx4], { z1.h - z4.h }, z5.h808; CHECK-NEXT:    ret809  call void @llvm.aarch64.sme.udot.single.za64.vg1x4.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4)810  %slice2 = add i32 %slice, 7811  call void @llvm.aarch64.sme.udot.single.za64.vg1x4.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4)812  ret void813}814 815define void @usdot_single_za32_u8_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2) #0 {816; CHECK-LABEL: usdot_single_za32_u8_vg1x2:817; CHECK:       // %bb.0:818; CHECK-NEXT:    mov w8, w0819; CHECK-NEXT:    usdot za.s[w8, 0, vgx2], { z1.b, z2.b }, z3.b820; CHECK-NEXT:    usdot za.s[w8, 7, vgx2], { z1.b, z2.b }, z3.b821; CHECK-NEXT:    ret822  call void @llvm.aarch64.sme.usdot.single.za32.vg1x2.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2)823  %slice2 = add i32 %slice, 7824  call void @llvm.aarch64.sme.usdot.single.za32.vg1x2.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2)825  ret void826}827 828define void @usdot_single_za32_u16_vg1x2_tuple(ptr %ptr, i64 %stride, <vscale x 16 x i8> %zn) #0 {829; CHECK-LABEL: usdot_single_za32_u16_vg1x2_tuple:830; CHECK:       // %bb.0: // %entry831; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill832; CHECK-NEXT:    addvl sp, sp, #-3833; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill834; CHECK-NEXT:    ptrue pn8.b835; CHECK-NEXT:    mov w8, wzr836; CHECK-NEXT:    str z10, [sp, #1, mul vl] // 16-byte Folded Spill837; CHECK-NEXT:    str z9, [sp, #2, mul vl] // 16-byte Folded Spill838; CHECK-NEXT:    ld1b { z1.b, z9.b }, pn8/z, [x0]839; CHECK-NEXT:    ld1b { z2.b, z10.b }, pn8/z, [x0, x1]840; CHECK-NEXT:    usdot za.s[w8, 0, vgx2], { z1.b, z2.b }, z0.b841; CHECK-NEXT:    usdot za.s[w8, 0, vgx2], { z9.b, z10.b }, z0.b842; CHECK-NEXT:    ldr z10, [sp, #1, mul vl] // 16-byte Folded Reload843; CHECK-NEXT:    ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload844; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload845; CHECK-NEXT:    addvl sp, sp, #3846; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload847; CHECK-NEXT:    ret848entry:849  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()850  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)851  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 0852  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 1853  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride854  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)855  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 0856  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 1857  call void @llvm.aarch64.sme.usdot.single.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> %zn)858  call void @llvm.aarch64.sme.usdot.single.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6, <vscale x 16 x i8> %zn)859  ret void860}861 862define void @usdot_single_za32_u8_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4) #0 {863; CHECK-LABEL: usdot_single_za32_u8_vg1x4:864; CHECK:       // %bb.0:865; CHECK-NEXT:    mov w8, w0866; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z1.b - z4.b }, z5.b867; CHECK-NEXT:    usdot za.s[w8, 7, vgx4], { z1.b - z4.b }, z5.b868; CHECK-NEXT:    ret869  call void @llvm.aarch64.sme.usdot.single.za32.vg1x4.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4)870  %slice2 = add i32 %slice, 7871  call void @llvm.aarch64.sme.usdot.single.za32.vg1x4.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4)872  ret void873}874 875define void @usdot_single_za32_u16_vg1x4_tuple(ptr %ptr, i64 %stride, <vscale x 16 x i8> %zn) #0 {876; CHECK-LABEL: usdot_single_za32_u16_vg1x4_tuple:877; CHECK:       // %bb.0: // %entry878; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill879; CHECK-NEXT:    addvl sp, sp, #-9880; CHECK-NEXT:    lsl x9, x1, #1881; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill882; CHECK-NEXT:    ptrue pn8.b883; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill884; CHECK-NEXT:    mov w8, wzr885; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill886; CHECK-NEXT:    add x10, x9, x1887; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill888; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill889; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill890; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill891; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill892; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill893; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]894; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]895; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]896; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]897; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b898; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b899; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b900; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b901; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload902; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload903; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload904; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload905; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload906; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload907; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload908; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload909; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload910; CHECK-NEXT:    addvl sp, sp, #9911; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload912; CHECK-NEXT:    ret913entry:914  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()915  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)916  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 0917  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 1918  %4 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 2919  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 3920  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride921  %6 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)922  %7 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 0923  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 1924  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 2925  %10 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 3926  %mul3 = shl i64 %stride, 1927  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3928  %11 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)929  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 0930  %13 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 1931  %14 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 2932  %15 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 3933  %mul5 = mul i64 %stride, 3934  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5935  %16 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)936  %17 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 0937  %18 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 1938  %19 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 2939  %20 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 3940  call void @llvm.aarch64.sme.usdot.single.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %7, <vscale x 16 x i8> %12, <vscale x 16 x i8> %17, <vscale x 16 x i8> %zn)941  call void @llvm.aarch64.sme.usdot.single.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> %zn)942  call void @llvm.aarch64.sme.usdot.single.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> %zn)943  call void @llvm.aarch64.sme.usdot.single.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> %zn)944  ret void945}946 947; == Multi, single (signed) ==948 949define void @sdot_single_za32_u16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2) #0 {950; CHECK-LABEL: sdot_single_za32_u16_vg1x2:951; CHECK:       // %bb.0:952; CHECK-NEXT:    mov w8, w0953; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z1.h, z2.h }, z3.h954; CHECK-NEXT:    sdot za.s[w8, 7, vgx2], { z1.h, z2.h }, z3.h955; CHECK-NEXT:    ret956  call void @llvm.aarch64.sme.sdot.single.za32.vg1x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2)957  %slice2 = add i32 %slice, 7958  call void @llvm.aarch64.sme.sdot.single.za32.vg1x2.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2)959  ret void960}961 962define void @sdot_single_za32_u16_vg1x2_tuple(ptr %ptr, i64 %stride, <vscale x 8 x i16> %zn) #0 {963; CHECK-LABEL: sdot_single_za32_u16_vg1x2_tuple:964; CHECK:       // %bb.0: // %entry965; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill966; CHECK-NEXT:    addvl sp, sp, #-3967; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill968; CHECK-NEXT:    ptrue pn8.b969; CHECK-NEXT:    add x9, x0, x1970; CHECK-NEXT:    str z10, [sp, #1, mul vl] // 16-byte Folded Spill971; CHECK-NEXT:    mov w8, wzr972; CHECK-NEXT:    str z9, [sp, #2, mul vl] // 16-byte Folded Spill973; CHECK-NEXT:    ld1h { z1.h, z9.h }, pn8/z, [x0]974; CHECK-NEXT:    ld1h { z2.h, z10.h }, pn8/z, [x9]975; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z1.h, z2.h }, z0.h976; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z9.h, z10.h }, z0.h977; CHECK-NEXT:    ldr z10, [sp, #1, mul vl] // 16-byte Folded Reload978; CHECK-NEXT:    ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload979; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload980; CHECK-NEXT:    addvl sp, sp, #3981; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload982; CHECK-NEXT:    ret983entry:984  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()985  %1 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") %0, ptr %ptr)986  %2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 0987  %3 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 1988  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride989  %4 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx2)990  %5 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %4, 0991  %6 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } %4, 1992  call void @llvm.aarch64.sme.sdot.single.za32.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %2, <vscale x 8 x i16> %5, <vscale x 8 x i16> %zn)993  call void @llvm.aarch64.sme.sdot.single.za32.vg1x2.nxv8i16(i32 0, <vscale x 8 x i16> %3, <vscale x 8 x i16> %6, <vscale x 8 x i16> %zn)994  ret void995}996 997define void @sdot_single_za32_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4) #0 {998; CHECK-LABEL: sdot_single_za32_u16_vg1x4:999; CHECK:       // %bb.0:1000; CHECK-NEXT:    mov w8, w01001; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z1.h - z4.h }, z5.h1002; CHECK-NEXT:    sdot za.s[w8, 7, vgx4], { z1.h - z4.h }, z5.h1003; CHECK-NEXT:    ret1004  call void @llvm.aarch64.sme.sdot.single.za32.vg1x4.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4)1005  %slice2 = add i32 %slice, 71006  call void @llvm.aarch64.sme.sdot.single.za32.vg1x4.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4)1007  ret void1008}1009 1010define void @sdot_single_za32_u16_vg1x4_tuple(ptr %ptr, i64 %stride, <vscale x 8 x i16> %zn) #0 {1011; CHECK-LABEL: sdot_single_za32_u16_vg1x4_tuple:1012; CHECK:       // %bb.0: // %entry1013; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill1014; CHECK-NEXT:    addvl sp, sp, #-91015; CHECK-NEXT:    add x9, x1, x1, lsl #11016; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill1017; CHECK-NEXT:    ptrue pn8.b1018; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill1019; CHECK-NEXT:    add x10, x0, x11020; CHECK-NEXT:    mov w8, wzr1021; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill1022; CHECK-NEXT:    add x9, x0, x91023; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill1024; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill1025; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill1026; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill1027; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill1028; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill1029; CHECK-NEXT:    ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0]1030; CHECK-NEXT:    ld1h { z17.h, z21.h, z25.h, z29.h }, pn8/z, [x10]1031; CHECK-NEXT:    ld1h { z18.h, z22.h, z26.h, z30.h }, pn8/z, [x0, x1, lsl #1]1032; CHECK-NEXT:    ld1h { z19.h, z23.h, z27.h, z31.h }, pn8/z, [x9]1033; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z16.h - z19.h }, z0.h1034; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z20.h - z23.h }, z0.h1035; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z24.h - z27.h }, z0.h1036; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z28.h - z31.h }, z0.h1037; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload1038; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload1039; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload1040; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload1041; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload1042; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload1043; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload1044; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload1045; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload1046; CHECK-NEXT:    addvl sp, sp, #91047; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload1048; CHECK-NEXT:    ret1049entry:1050  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()1051  %1 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %ptr)1052  %2 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 01053  %3 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 11054  %4 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 21055  %5 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %1, 31056  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride1057  %6 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx2)1058  %7 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 01059  %8 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 11060  %9 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 21061  %10 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %6, 31062  %mul3 = shl i64 %stride, 11063  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul31064  %11 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx4)1065  %12 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %11, 01066  %13 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %11, 11067  %14 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %11, 21068  %15 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %11, 31069  %mul5 = mul i64 %stride, 31070  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul51071  %16 = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %0, ptr %arrayidx6)1072  %17 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %16, 01073  %18 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %16, 11074  %19 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %16, 21075  %20 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %16, 31076  call void @llvm.aarch64.sme.sdot.single.za32.vg1x4.nxv8i16(i32 0, <vscale x 8 x i16> %2, <vscale x 8 x i16> %7, <vscale x 8 x i16> %12, <vscale x 8 x i16> %17, <vscale x 8 x i16> %zn)1077  call void @llvm.aarch64.sme.sdot.single.za32.vg1x4.nxv8i16(i32 0, <vscale x 8 x i16> %3, <vscale x 8 x i16> %8, <vscale x 8 x i16> %13, <vscale x 8 x i16> %18, <vscale x 8 x i16> %zn)1078  call void @llvm.aarch64.sme.sdot.single.za32.vg1x4.nxv8i16(i32 0, <vscale x 8 x i16> %4, <vscale x 8 x i16> %9, <vscale x 8 x i16> %14, <vscale x 8 x i16> %19, <vscale x 8 x i16> %zn)1079  call void @llvm.aarch64.sme.sdot.single.za32.vg1x4.nxv8i16(i32 0, <vscale x 8 x i16> %5, <vscale x 8 x i16> %10, <vscale x 8 x i16> %15, <vscale x 8 x i16> %20, <vscale x 8 x i16> %zn)1080  ret void1081}1082 1083define void @sdot_single_za32_u8_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2) #0 {1084; CHECK-LABEL: sdot_single_za32_u8_vg1x2:1085; CHECK:       // %bb.0:1086; CHECK-NEXT:    mov w8, w01087; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z1.b, z2.b }, z3.b1088; CHECK-NEXT:    sdot za.s[w8, 7, vgx2], { z1.b, z2.b }, z3.b1089; CHECK-NEXT:    ret1090  call void @llvm.aarch64.sme.sdot.single.za32.vg1x2.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2)1091  %slice2 = add i32 %slice, 71092  call void @llvm.aarch64.sme.sdot.single.za32.vg1x2.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2)1093  ret void1094}1095 1096define void @sdot_single_za32_u8_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4) #0 {1097; CHECK-LABEL: sdot_single_za32_u8_vg1x4:1098; CHECK:       // %bb.0:1099; CHECK-NEXT:    mov w8, w01100; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z1.b - z4.b }, z5.b1101; CHECK-NEXT:    sdot za.s[w8, 7, vgx4], { z1.b - z4.b }, z5.b1102; CHECK-NEXT:    ret1103  call void @llvm.aarch64.sme.sdot.single.za32.vg1x4.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4)1104  %slice2 = add i32 %slice, 71105  call void @llvm.aarch64.sme.sdot.single.za32.vg1x4.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4)1106  ret void1107}1108 1109define void @sdot_single_za64_u16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2) #1 {1110; CHECK-LABEL: sdot_single_za64_u16_vg1x2:1111; CHECK:       // %bb.0:1112; CHECK-NEXT:    mov w8, w01113; CHECK-NEXT:    sdot za.d[w8, 0, vgx2], { z1.h, z2.h }, z3.h1114; CHECK-NEXT:    sdot za.d[w8, 7, vgx2], { z1.h, z2.h }, z3.h1115; CHECK-NEXT:    ret1116  call void @llvm.aarch64.sme.sdot.single.za64.vg1x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2)1117  %slice2 = add i32 %slice, 71118  call void @llvm.aarch64.sme.sdot.single.za64.vg1x2.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2)1119  ret void1120}1121 1122define void @sdot_single_za64_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4) #1 {1123; CHECK-LABEL: sdot_single_za64_u16_vg1x4:1124; CHECK:       // %bb.0:1125; CHECK-NEXT:    mov w8, w01126; CHECK-NEXT:    sdot za.d[w8, 0, vgx4], { z1.h - z4.h }, z5.h1127; CHECK-NEXT:    sdot za.d[w8, 7, vgx4], { z1.h - z4.h }, z5.h1128; CHECK-NEXT:    ret1129  call void @llvm.aarch64.sme.sdot.single.za64.vg1x4.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4)1130  %slice2 = add i32 %slice, 71131  call void @llvm.aarch64.sme.sdot.single.za64.vg1x4.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4)1132  ret void1133}1134 1135define void @sudot_single_za32_u8_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2) #0 {1136; CHECK-LABEL: sudot_single_za32_u8_vg1x2:1137; CHECK:       // %bb.0:1138; CHECK-NEXT:    mov w8, w01139; CHECK-NEXT:    sudot za.s[w8, 0, vgx2], { z1.b, z2.b }, z3.b1140; CHECK-NEXT:    sudot za.s[w8, 7, vgx2], { z1.b, z2.b }, z3.b1141; CHECK-NEXT:    ret1142  call void @llvm.aarch64.sme.sudot.single.za32.vg1x2.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2)1143  %slice2 = add i32 %slice, 71144  call void @llvm.aarch64.sme.sudot.single.za32.vg1x2.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2)1145  ret void1146}1147 1148define void @sudot_single_za32_u16_vg1x2_tuple(ptr %ptr, i64 %stride, <vscale x 16 x i8> %zn) #0 {1149; CHECK-LABEL: sudot_single_za32_u16_vg1x2_tuple:1150; CHECK:       // %bb.0: // %entry1151; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill1152; CHECK-NEXT:    addvl sp, sp, #-31153; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill1154; CHECK-NEXT:    ptrue pn8.b1155; CHECK-NEXT:    mov w8, wzr1156; CHECK-NEXT:    str z10, [sp, #1, mul vl] // 16-byte Folded Spill1157; CHECK-NEXT:    str z9, [sp, #2, mul vl] // 16-byte Folded Spill1158; CHECK-NEXT:    ld1b { z1.b, z9.b }, pn8/z, [x0]1159; CHECK-NEXT:    ld1b { z2.b, z10.b }, pn8/z, [x0, x1]1160; CHECK-NEXT:    sudot za.s[w8, 0, vgx2], { z1.b, z2.b }, z0.b1161; CHECK-NEXT:    sudot za.s[w8, 0, vgx2], { z9.b, z10.b }, z0.b1162; CHECK-NEXT:    ldr z10, [sp, #1, mul vl] // 16-byte Folded Reload1163; CHECK-NEXT:    ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload1164; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload1165; CHECK-NEXT:    addvl sp, sp, #31166; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload1167; CHECK-NEXT:    ret1168entry:1169  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()1170  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)1171  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 01172  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 11173  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride1174  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)1175  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 01176  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 11177  call void @llvm.aarch64.sme.sudot.single.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> %zn)1178  call void @llvm.aarch64.sme.sudot.single.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6, <vscale x 16 x i8> %zn)1179  ret void1180}1181 1182define void @sudot_single_za32_u8_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4) #0 {1183; CHECK-LABEL: sudot_single_za32_u8_vg1x4:1184; CHECK:       // %bb.0:1185; CHECK-NEXT:    mov w8, w01186; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z1.b - z4.b }, z5.b1187; CHECK-NEXT:    sudot za.s[w8, 7, vgx4], { z1.b - z4.b }, z5.b1188; CHECK-NEXT:    ret1189  call void @llvm.aarch64.sme.sudot.single.za32.vg1x4.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4)1190  %slice2 = add i32 %slice, 71191  call void @llvm.aarch64.sme.sudot.single.za32.vg1x4.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4)1192  ret void1193}1194 1195define void @sudot_single_za32_u16_vg1x4_tuple(ptr %ptr, i64 %stride, <vscale x 16 x i8> %zn) #0 {1196; CHECK-LABEL: sudot_single_za32_u16_vg1x4_tuple:1197; CHECK:       // %bb.0: // %entry1198; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill1199; CHECK-NEXT:    addvl sp, sp, #-91200; CHECK-NEXT:    lsl x9, x1, #11201; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill1202; CHECK-NEXT:    ptrue pn8.b1203; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill1204; CHECK-NEXT:    mov w8, wzr1205; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill1206; CHECK-NEXT:    add x10, x9, x11207; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill1208; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill1209; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill1210; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill1211; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill1212; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill1213; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]1214; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]1215; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]1216; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]1217; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b1218; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b1219; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b1220; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b1221; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload1222; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload1223; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload1224; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload1225; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload1226; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload1227; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload1228; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload1229; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload1230; CHECK-NEXT:    addvl sp, sp, #91231; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload1232; CHECK-NEXT:    ret1233entry:1234  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()1235  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)1236  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 01237  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 11238  %4 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 21239  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 31240  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride1241  %6 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)1242  %7 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 01243  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 11244  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 21245  %10 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 31246  %mul3 = shl i64 %stride, 11247  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul31248  %11 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)1249  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 01250  %13 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 11251  %14 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 21252  %15 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 31253  %mul5 = mul i64 %stride, 31254  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul51255  %16 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)1256  %17 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 01257  %18 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 11258  %19 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 21259  %20 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 31260  call void @llvm.aarch64.sme.sudot.single.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %7, <vscale x 16 x i8> %12, <vscale x 16 x i8> %17, <vscale x 16 x i8> %zn)1261  call void @llvm.aarch64.sme.sudot.single.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> %zn)1262  call void @llvm.aarch64.sme.sudot.single.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> %zn)1263  call void @llvm.aarch64.sme.sudot.single.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> %zn)1264  ret void1265}1266 1267; == Multi, indexed (unsigned) ==1268 1269define void @udot_lane_za32_u16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2) #0 {1270; CHECK-LABEL: udot_lane_za32_u16_vg1x2:1271; CHECK:       // %bb.0:1272; CHECK-NEXT:    mov z5.d, z2.d1273; CHECK-NEXT:    mov z4.d, z1.d1274; CHECK-NEXT:    mov w8, w01275; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z4.h, z5.h }, z3.h[3]1276; CHECK-NEXT:    udot za.s[w8, 7, vgx2], { z4.h, z5.h }, z3.h[3]1277; CHECK-NEXT:    ret1278  call void @llvm.aarch64.sme.udot.lane.za32.vg1x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, i32 3)1279  %slice2 = add i32 %slice, 71280  call void @llvm.aarch64.sme.udot.lane.za32.vg1x2.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, i32 3)1281  ret void1282}1283 1284define void @udot_lane_za32_u16_vg1x4(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4) #0 {1285; CHECK-LABEL: udot_lane_za32_u16_vg1x4:1286; CHECK:       // %bb.0:1287; CHECK-NEXT:    mov w8, w01288; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z0.h - z3.h }, z4.h[3]1289; CHECK-NEXT:    udot za.s[w8, 7, vgx4], { z0.h - z3.h }, z4.h[3]1290; CHECK-NEXT:    ret1291  call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,1292                                                           <vscale x 8 x i16> %zn4, i32 3)1293  %slice2 = add i32 %slice, 71294  call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,1295                                                           <vscale x 8 x i16> %zn4, i32 3)1296  ret void1297}1298 1299define void @udot_lane_za32_u8_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2) #0 {1300; CHECK-LABEL: udot_lane_za32_u8_vg1x2:1301; CHECK:       // %bb.0:1302; CHECK-NEXT:    mov z5.d, z2.d1303; CHECK-NEXT:    mov z4.d, z1.d1304; CHECK-NEXT:    mov w8, w01305; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z4.b, z5.b }, z3.b[3]1306; CHECK-NEXT:    udot za.s[w8, 7, vgx2], { z4.b, z5.b }, z3.b[3]1307; CHECK-NEXT:    ret1308  call void @llvm.aarch64.sme.udot.lane.za32.vg1x2.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, i32 3)1309  %slice2 = add i32 %slice, 71310  call void @llvm.aarch64.sme.udot.lane.za32.vg1x2.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, i32 3)1311  ret void1312}1313 1314define void @udot_lane_za32_u8_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4) #0 {1315; CHECK-LABEL: udot_lane_za32_u8_vg1x4:1316; CHECK:       // %bb.0:1317; CHECK-NEXT:    mov z27.d, z4.d1318; CHECK-NEXT:    mov z26.d, z3.d1319; CHECK-NEXT:    mov w8, w01320; CHECK-NEXT:    mov z25.d, z2.d1321; CHECK-NEXT:    mov z24.d, z1.d1322; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z24.b - z27.b }, z5.b[3]1323; CHECK-NEXT:    udot za.s[w8, 7, vgx4], { z24.b - z27.b }, z5.b[3]1324; CHECK-NEXT:    ret1325  call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,1326                                                           <vscale x 16 x i8> %zn4, i32 3)1327  %slice2 = add i32 %slice, 71328  call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,1329                                                           <vscale x 16 x i8> %zn4, i32 3)1330  ret void1331}1332 1333define void @udot_form_2x_tuple(ptr %ptr, i64 %stride) #0 {1334; CHECK-LABEL: udot_form_2x_tuple:1335; CHECK:       // %bb.0: // %entry1336; CHECK-NEXT:    ptrue pn8.b1337; CHECK-NEXT:    mov w8, wzr1338; CHECK-NEXT:    ld1b { z16.b, z24.b }, pn8/z, [x0]1339; CHECK-NEXT:    ld1b { z17.b, z25.b }, pn8/z, [x0, x1]1340; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z16.b, z17.b }, z0.b[0]1341; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z24.b, z25.b }, z0.b[0]1342; CHECK-NEXT:    ret1343entry:1344  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()1345  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)1346  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 01347  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 11348  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride1349  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)1350  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 01351  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 11352  tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> poison, i32 0)1353  tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6, <vscale x 16 x i8> poison, i32 0)1354  ret void1355}1356 1357define void @udot_form_2x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8> %scalable_arg) #0 {1358; CHECK-LABEL: udot_form_2x_tuple_svecc:1359; CHECK:       // %bb.0: // %entry1360; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill1361; CHECK-NEXT:    addvl sp, sp, #-31362; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill1363; CHECK-NEXT:    ptrue pn8.b1364; CHECK-NEXT:    mov w8, wzr1365; CHECK-NEXT:    str z11, [sp, #1, mul vl] // 16-byte Folded Spill1366; CHECK-NEXT:    str z10, [sp, #2, mul vl] // 16-byte Folded Spill1367; CHECK-NEXT:    ld1b { z2.b, z10.b }, pn8/z, [x0]1368; CHECK-NEXT:    ld1b { z3.b, z11.b }, pn8/z, [x0, x1]1369; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z2.b, z3.b }, z0.b[0]1370; CHECK-NEXT:    udot za.s[w8, 0, vgx2], { z10.b, z11.b }, z0.b[0]1371; CHECK-NEXT:    ldr z11, [sp, #1, mul vl] // 16-byte Folded Reload1372; CHECK-NEXT:    ldr z10, [sp, #2, mul vl] // 16-byte Folded Reload1373; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload1374; CHECK-NEXT:    str z0, [x0]1375; CHECK-NEXT:    addvl sp, sp, #31376; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload1377; CHECK-NEXT:    ret1378entry:1379  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()1380  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)1381  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 01382  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 11383  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride1384  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)1385  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 01386  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 11387  tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> poison, i32 0)1388  tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6, <vscale x 16 x i8> poison, i32 0)1389  store <vscale x 16 x i8> %scalable_arg, ptr %ptr1390  ret void1391}1392 1393define void @udot_form_4x_tuple(ptr %ptr, i64 %stride) #0 {1394; CHECK-LABEL: udot_form_4x_tuple:1395; CHECK:       // %bb.0: // %entry1396; CHECK-NEXT:    lsl x9, x1, #11397; CHECK-NEXT:    ptrue pn8.b1398; CHECK-NEXT:    mov w8, wzr1399; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]1400; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]1401; CHECK-NEXT:    add x10, x9, x11402; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]1403; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]1404; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]1405; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]1406; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]1407; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]1408; CHECK-NEXT:    ret1409entry:1410  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()1411  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)1412  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 01413  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 11414  %4 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 21415  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 31416  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride1417  %6 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)1418  %7 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 01419  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 11420  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 21421  %10 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 31422  %mul3 = shl i64 %stride, 11423  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul31424  %11 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)1425  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 01426  %13 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 11427  %14 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 21428  %15 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 31429  %mul5 = mul i64 %stride, 31430  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul51431  %16 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)1432  %17 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 01433  %18 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 11434  %19 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 21435  %20 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 31436  tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %7, <vscale x 16 x i8> %12, <vscale x 16 x i8> %17, <vscale x 16 x i8> poison, i32 0)1437  tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> poison, i32 0)1438  tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> poison, i32 0)1439  tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> poison, i32 0)1440  ret void1441}1442 1443define void @udot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8> %scalable_arg) #0 {1444; CHECK-LABEL: udot_form_4x_tuple_svecc:1445; CHECK:       // %bb.0: // %entry1446; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill1447; CHECK-NEXT:    addvl sp, sp, #-91448; CHECK-NEXT:    lsl x9, x1, #11449; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill1450; CHECK-NEXT:    ptrue pn8.b1451; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill1452; CHECK-NEXT:    mov w8, wzr1453; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill1454; CHECK-NEXT:    add x10, x9, x11455; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill1456; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill1457; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill1458; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill1459; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill1460; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill1461; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]1462; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]1463; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]1464; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]1465; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]1466; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]1467; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]1468; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]1469; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload1470; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload1471; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload1472; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload1473; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload1474; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload1475; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload1476; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload1477; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload1478; CHECK-NEXT:    str z0, [x0]1479; CHECK-NEXT:    addvl sp, sp, #91480; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload1481; CHECK-NEXT:    ret1482entry:1483  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()1484  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)1485  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 01486  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 11487  %4 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 21488  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 31489  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride1490  %6 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)1491  %7 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 01492  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 11493  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 21494  %10 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 31495  %mul3 = shl i64 %stride, 11496  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul31497  %11 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)1498  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 01499  %13 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 11500  %14 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 21501  %15 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 31502  %mul5 = mul i64 %stride, 31503  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul51504  %16 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)1505  %17 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 01506  %18 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 11507  %19 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 21508  %20 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 31509  tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %7, <vscale x 16 x i8> %12, <vscale x 16 x i8> %17, <vscale x 16 x i8> poison, i32 0)1510  tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> poison, i32 0)1511  tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> poison, i32 0)1512  tail call void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> poison, i32 0)1513  store <vscale x 16 x i8> %scalable_arg, ptr %ptr1514  ret void1515}1516 1517define void @udot_single_za32_u16_vg1x4_x2load_x4tuple(ptr %ptr, i64 %stride, <vscale x 16 x i8> %zn) #0 {1518; CHECK-LABEL: udot_single_za32_u16_vg1x4_x2load_x4tuple:1519; CHECK:       // %bb.0: // %entry1520; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill1521; CHECK-NEXT:    addvl sp, sp, #-51522; CHECK-NEXT:    lsl x9, x1, #11523; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill1524; CHECK-NEXT:    ptrue pn8.b1525; CHECK-NEXT:    str z12, [sp, #1, mul vl] // 16-byte Folded Spill1526; CHECK-NEXT:    st1b { z10.b, z11.b }, pn8, [sp, #2, mul vl] // 32-byte Folded Spill1527; CHECK-NEXT:    ptrue pn8.b1528; CHECK-NEXT:    str z9, [sp, #4, mul vl] // 16-byte Folded Spill1529; CHECK-NEXT:    add x10, x9, x11530; CHECK-NEXT:    mov w8, wzr1531; CHECK-NEXT:    ld1b { z1.b, z9.b }, pn8/z, [x0]1532; CHECK-NEXT:    ld1b { z2.b, z10.b }, pn8/z, [x0, x1]1533; CHECK-NEXT:    ld1b { z3.b, z11.b }, pn8/z, [x0, x9]1534; CHECK-NEXT:    ld1b { z4.b, z12.b }, pn8/z, [x0, x10]1535; CHECK-NEXT:    ptrue pn8.b1536; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z1.b - z4.b }, z0.b1537; CHECK-NEXT:    udot za.s[w8, 0, vgx4], { z9.b - z12.b }, z0.b1538; CHECK-NEXT:    ldr z12, [sp, #1, mul vl] // 16-byte Folded Reload1539; CHECK-NEXT:    ld1b { z10.b, z11.b }, pn8/z, [sp, #2, mul vl] // 32-byte Folded Reload1540; CHECK-NEXT:    ldr z9, [sp, #4, mul vl] // 16-byte Folded Reload1541; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload1542; CHECK-NEXT:    addvl sp, sp, #51543; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload1544; CHECK-NEXT:    ret1545entry:1546  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()1547  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)1548  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 01549  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 11550  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride1551  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)1552  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 01553  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 11554  %mul3 = shl i64 %stride, 11555  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul31556  %7 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)1557  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %7, 01558  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %7, 11559  %mul5 = mul i64 %stride, 31560  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul51561  %10 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)1562  %11 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %10, 01563  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %10, 11564  call void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> %8, <vscale x 16 x i8> %11, <vscale x 16 x i8> %zn)1565  call void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6, <vscale x 16 x i8> %9, <vscale x 16 x i8> %12, <vscale x 16 x i8> %zn)1566  ret void1567}1568 1569define void @udot_lane_za64_u16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2) #1 {1570; CHECK-LABEL: udot_lane_za64_u16_vg1x2:1571; CHECK:       // %bb.0:1572; CHECK-NEXT:    mov z5.d, z2.d1573; CHECK-NEXT:    mov z4.d, z1.d1574; CHECK-NEXT:    mov w8, w01575; CHECK-NEXT:    udot za.d[w8, 0, vgx2], { z4.h, z5.h }, z3.h[1]1576; CHECK-NEXT:    udot za.d[w8, 7, vgx2], { z4.h, z5.h }, z3.h[1]1577; CHECK-NEXT:    ret1578  call void @llvm.aarch64.sme.udot.lane.za64.vg1x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, i32 1)1579  %slice2 = add i32 %slice, 71580  call void @llvm.aarch64.sme.udot.lane.za64.vg1x2.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, i32 1)1581  ret void1582}1583 1584define void @udot_lane_za64_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4) #1 {1585; CHECK-LABEL: udot_lane_za64_u16_vg1x4:1586; CHECK:       // %bb.0:1587; CHECK-NEXT:    mov z27.d, z4.d1588; CHECK-NEXT:    mov z26.d, z3.d1589; CHECK-NEXT:    mov w8, w01590; CHECK-NEXT:    mov z25.d, z2.d1591; CHECK-NEXT:    mov z24.d, z1.d1592; CHECK-NEXT:    udot za.d[w8, 0, vgx4], { z24.h - z27.h }, z5.h[1]1593; CHECK-NEXT:    udot za.d[w8, 7, vgx4], { z24.h - z27.h }, z5.h[1]1594; CHECK-NEXT:    ret1595  call void @llvm.aarch64.sme.udot.lane.za64.vg1x4.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,1596                                                           <vscale x 8 x i16> %zn4, i32 1)1597  %slice2 = add i32 %slice, 71598  call void @llvm.aarch64.sme.udot.lane.za64.vg1x4.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,1599                                                           <vscale x 8 x i16> %zn4, i32 1)1600  ret void1601}1602 1603define void @usdot_lane_za32_u8_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2) #0 {1604; CHECK-LABEL: usdot_lane_za32_u8_vg1x2:1605; CHECK:       // %bb.0:1606; CHECK-NEXT:    mov z5.d, z2.d1607; CHECK-NEXT:    mov z4.d, z1.d1608; CHECK-NEXT:    mov w8, w01609; CHECK-NEXT:    usdot za.s[w8, 0, vgx2], { z4.b, z5.b }, z3.b[3]1610; CHECK-NEXT:    usdot za.s[w8, 7, vgx2], { z4.b, z5.b }, z3.b[3]1611; CHECK-NEXT:    ret1612  call void @llvm.aarch64.sme.usdot.lane.za32.vg1x2.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, i32 3)1613  %slice2 = add i32 %slice, 71614  call void @llvm.aarch64.sme.usdot.lane.za32.vg1x2.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, i32 3)1615  ret void1616}1617 1618define void @usdot_lane_za32_u8_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4) #0 {1619; CHECK-LABEL: usdot_lane_za32_u8_vg1x4:1620; CHECK:       // %bb.0:1621; CHECK-NEXT:    mov z27.d, z4.d1622; CHECK-NEXT:    mov z26.d, z3.d1623; CHECK-NEXT:    mov w8, w01624; CHECK-NEXT:    mov z25.d, z2.d1625; CHECK-NEXT:    mov z24.d, z1.d1626; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z24.b - z27.b }, z5.b[3]1627; CHECK-NEXT:    usdot za.s[w8, 7, vgx4], { z24.b - z27.b }, z5.b[3]1628; CHECK-NEXT:    ret1629  call void @llvm.aarch64.sme.usdot.lane.za32.vg1x4.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,1630                                                            <vscale x 16 x i8> %zn4, i32 3)1631  %slice2 = add i32 %slice, 71632  call void @llvm.aarch64.sme.usdot.lane.za32.vg1x4.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,1633                                                            <vscale x 16 x i8> %zn4, i32 3)1634  ret void1635}1636 1637define void @usdot_form_2x_tuple(ptr %ptr, i64 %stride) #0 {1638; CHECK-LABEL: usdot_form_2x_tuple:1639; CHECK:       // %bb.0: // %entry1640; CHECK-NEXT:    ptrue pn8.b1641; CHECK-NEXT:    mov w8, wzr1642; CHECK-NEXT:    ld1b { z16.b, z24.b }, pn8/z, [x0]1643; CHECK-NEXT:    ld1b { z17.b, z25.b }, pn8/z, [x0, x1]1644; CHECK-NEXT:    usdot za.s[w8, 0, vgx2], { z16.b, z17.b }, z0.b[0]1645; CHECK-NEXT:    usdot za.s[w8, 0, vgx2], { z24.b, z25.b }, z0.b[0]1646; CHECK-NEXT:    ret1647entry:1648  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()1649  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)1650  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 01651  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 11652  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride1653  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)1654  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 01655  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 11656  tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> poison, i32 0)1657  tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6, <vscale x 16 x i8> poison, i32 0)1658  ret void1659}1660 1661define void @usdot_form_2x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8> %scalable_arg) #0 {1662; CHECK-LABEL: usdot_form_2x_tuple_svecc:1663; CHECK:       // %bb.0: // %entry1664; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill1665; CHECK-NEXT:    addvl sp, sp, #-31666; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill1667; CHECK-NEXT:    ptrue pn8.b1668; CHECK-NEXT:    mov w8, wzr1669; CHECK-NEXT:    str z11, [sp, #1, mul vl] // 16-byte Folded Spill1670; CHECK-NEXT:    str z10, [sp, #2, mul vl] // 16-byte Folded Spill1671; CHECK-NEXT:    ld1b { z2.b, z10.b }, pn8/z, [x0]1672; CHECK-NEXT:    ld1b { z3.b, z11.b }, pn8/z, [x0, x1]1673; CHECK-NEXT:    usdot za.s[w8, 0, vgx2], { z2.b, z3.b }, z0.b[0]1674; CHECK-NEXT:    usdot za.s[w8, 0, vgx2], { z10.b, z11.b }, z0.b[0]1675; CHECK-NEXT:    ldr z11, [sp, #1, mul vl] // 16-byte Folded Reload1676; CHECK-NEXT:    ldr z10, [sp, #2, mul vl] // 16-byte Folded Reload1677; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload1678; CHECK-NEXT:    str z0, [x0]1679; CHECK-NEXT:    addvl sp, sp, #31680; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload1681; CHECK-NEXT:    ret1682entry:1683  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()1684  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)1685  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 01686  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 11687  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride1688  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)1689  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 01690  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 11691  tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> poison, i32 0)1692  tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6, <vscale x 16 x i8> poison, i32 0)1693  store <vscale x 16 x i8> %scalable_arg, ptr %ptr1694  ret void1695}1696 1697define void @usdot_form_4x_tuple(ptr %ptr, i64 %stride) #0 {1698; CHECK-LABEL: usdot_form_4x_tuple:1699; CHECK:       // %bb.0: // %entry1700; CHECK-NEXT:    lsl x9, x1, #11701; CHECK-NEXT:    ptrue pn8.b1702; CHECK-NEXT:    mov w8, wzr1703; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]1704; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]1705; CHECK-NEXT:    add x10, x9, x11706; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]1707; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]1708; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]1709; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]1710; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]1711; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]1712; CHECK-NEXT:    ret1713entry:1714  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()1715  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)1716  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 01717  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 11718  %4 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 21719  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 31720  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride1721  %6 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)1722  %7 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 01723  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 11724  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 21725  %10 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 31726  %mul3 = shl i64 %stride, 11727  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul31728  %11 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)1729  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 01730  %13 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 11731  %14 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 21732  %15 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 31733  %mul5 = mul i64 %stride, 31734  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul51735  %16 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)1736  %17 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 01737  %18 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 11738  %19 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 21739  %20 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 31740  tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %7, <vscale x 16 x i8> %12, <vscale x 16 x i8> %17, <vscale x 16 x i8> poison, i32 0)1741  tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> poison, i32 0)1742  tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> poison, i32 0)1743  tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> poison, i32 0)1744  ret void1745}1746 1747define void @usdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8> %scalable_arg) #0 {1748; CHECK-LABEL: usdot_form_4x_tuple_svecc:1749; CHECK:       // %bb.0: // %entry1750; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill1751; CHECK-NEXT:    addvl sp, sp, #-91752; CHECK-NEXT:    lsl x9, x1, #11753; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill1754; CHECK-NEXT:    ptrue pn8.b1755; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill1756; CHECK-NEXT:    mov w8, wzr1757; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill1758; CHECK-NEXT:    add x10, x9, x11759; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill1760; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill1761; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill1762; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill1763; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill1764; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill1765; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]1766; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]1767; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]1768; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]1769; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]1770; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]1771; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]1772; CHECK-NEXT:    usdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]1773; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload1774; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload1775; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload1776; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload1777; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload1778; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload1779; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload1780; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload1781; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload1782; CHECK-NEXT:    str z0, [x0]1783; CHECK-NEXT:    addvl sp, sp, #91784; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload1785; CHECK-NEXT:    ret1786entry:1787  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()1788  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)1789  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 01790  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 11791  %4 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 21792  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 31793  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride1794  %6 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)1795  %7 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 01796  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 11797  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 21798  %10 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 31799  %mul3 = shl i64 %stride, 11800  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul31801  %11 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)1802  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 01803  %13 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 11804  %14 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 21805  %15 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 31806  %mul5 = mul i64 %stride, 31807  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul51808  %16 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)1809  %17 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 01810  %18 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 11811  %19 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 21812  %20 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 31813  tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %7, <vscale x 16 x i8> %12, <vscale x 16 x i8> %17, <vscale x 16 x i8> poison, i32 0)1814  tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> poison, i32 0)1815  tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> poison, i32 0)1816  tail call void @llvm.aarch64.sme.usdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> poison, i32 0)1817  store <vscale x 16 x i8> %scalable_arg, ptr %ptr1818  ret void1819}1820 1821; == Multi, indexed (signed) ==1822 1823define void @sdot_lane_za32_u16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2) #0 {1824; CHECK-LABEL: sdot_lane_za32_u16_vg1x2:1825; CHECK:       // %bb.0:1826; CHECK-NEXT:    mov z5.d, z2.d1827; CHECK-NEXT:    mov z4.d, z1.d1828; CHECK-NEXT:    mov w8, w01829; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z4.h, z5.h }, z3.h[3]1830; CHECK-NEXT:    sdot za.s[w8, 7, vgx2], { z4.h, z5.h }, z3.h[3]1831; CHECK-NEXT:    ret1832  call void @llvm.aarch64.sme.sdot.lane.za32.vg1x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, i32 3)1833  %slice2 = add i32 %slice, 71834  call void @llvm.aarch64.sme.sdot.lane.za32.vg1x2.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, i32 3)1835  ret void1836}1837 1838define void @sdot_lane_za32_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4) #0 {1839; CHECK-LABEL: sdot_lane_za32_u16_vg1x4:1840; CHECK:       // %bb.0:1841; CHECK-NEXT:    mov z27.d, z4.d1842; CHECK-NEXT:    mov z26.d, z3.d1843; CHECK-NEXT:    mov w8, w01844; CHECK-NEXT:    mov z25.d, z2.d1845; CHECK-NEXT:    mov z24.d, z1.d1846; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z24.h - z27.h }, z5.h[3]1847; CHECK-NEXT:    sdot za.s[w8, 7, vgx4], { z24.h - z27.h }, z5.h[3]1848; CHECK-NEXT:    ret1849  call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,1850                                                           <vscale x 8 x i16> %zn4, i32 3)1851  %slice2 = add i32 %slice, 71852  call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,1853                                                           <vscale x 8 x i16> %zn4, i32 3)1854  ret void1855}1856 1857define void @sdot_lane_za32_u8_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2) #0 {1858; CHECK-LABEL: sdot_lane_za32_u8_vg1x2:1859; CHECK:       // %bb.0:1860; CHECK-NEXT:    mov z5.d, z2.d1861; CHECK-NEXT:    mov z4.d, z1.d1862; CHECK-NEXT:    mov w8, w01863; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z4.b, z5.b }, z3.b[3]1864; CHECK-NEXT:    sdot za.s[w8, 7, vgx2], { z4.b, z5.b }, z3.b[3]1865; CHECK-NEXT:    ret1866  call void @llvm.aarch64.sme.sdot.lane.za32.vg1x2.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, i32 3)1867  %slice2 = add i32 %slice, 71868  call void @llvm.aarch64.sme.sdot.lane.za32.vg1x2.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, i32 3)1869  ret void1870}1871 1872define void @sdot_lane_za32_u8_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4) #0 {1873; CHECK-LABEL: sdot_lane_za32_u8_vg1x4:1874; CHECK:       // %bb.0:1875; CHECK-NEXT:    mov z27.d, z4.d1876; CHECK-NEXT:    mov z26.d, z3.d1877; CHECK-NEXT:    mov w8, w01878; CHECK-NEXT:    mov z25.d, z2.d1879; CHECK-NEXT:    mov z24.d, z1.d1880; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z24.b - z27.b }, z5.b[3]1881; CHECK-NEXT:    sdot za.s[w8, 7, vgx4], { z24.b - z27.b }, z5.b[3]1882; CHECK-NEXT:    ret1883  call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,1884                                                           <vscale x 16 x i8> %zn4, i32 3)1885  %slice2 = add i32 %slice, 71886  call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,1887                                                           <vscale x 16 x i8> %zn4, i32 3)1888  ret void1889}1890 1891define void @sdot_form_2x_tuple(ptr %ptr, i64 %stride) #0 {1892; CHECK-LABEL: sdot_form_2x_tuple:1893; CHECK:       // %bb.0: // %entry1894; CHECK-NEXT:    ptrue pn8.b1895; CHECK-NEXT:    mov w8, wzr1896; CHECK-NEXT:    ld1b { z16.b, z24.b }, pn8/z, [x0]1897; CHECK-NEXT:    ld1b { z17.b, z25.b }, pn8/z, [x0, x1]1898; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z16.b, z17.b }, z0.b[0]1899; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z24.b, z25.b }, z0.b[0]1900; CHECK-NEXT:    ret1901entry:1902  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()1903  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)1904  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 01905  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 11906  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride1907  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)1908  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 01909  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 11910  tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> poison, i32 0)1911  tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6, <vscale x 16 x i8> poison, i32 0)1912  ret void1913}1914 1915define void @sdot_form_2x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8> %scalable_arg) #0 {1916; CHECK-LABEL: sdot_form_2x_tuple_svecc:1917; CHECK:       // %bb.0: // %entry1918; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill1919; CHECK-NEXT:    addvl sp, sp, #-31920; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill1921; CHECK-NEXT:    ptrue pn8.b1922; CHECK-NEXT:    mov w8, wzr1923; CHECK-NEXT:    str z11, [sp, #1, mul vl] // 16-byte Folded Spill1924; CHECK-NEXT:    str z10, [sp, #2, mul vl] // 16-byte Folded Spill1925; CHECK-NEXT:    ld1b { z2.b, z10.b }, pn8/z, [x0]1926; CHECK-NEXT:    ld1b { z3.b, z11.b }, pn8/z, [x0, x1]1927; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z2.b, z3.b }, z0.b[0]1928; CHECK-NEXT:    sdot za.s[w8, 0, vgx2], { z10.b, z11.b }, z0.b[0]1929; CHECK-NEXT:    ldr z11, [sp, #1, mul vl] // 16-byte Folded Reload1930; CHECK-NEXT:    ldr z10, [sp, #2, mul vl] // 16-byte Folded Reload1931; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload1932; CHECK-NEXT:    str z0, [x0]1933; CHECK-NEXT:    addvl sp, sp, #31934; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload1935; CHECK-NEXT:    ret1936entry:1937  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()1938  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)1939  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 01940  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 11941  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride1942  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)1943  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 01944  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 11945  tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> poison, i32 0)1946  tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6, <vscale x 16 x i8> poison, i32 0)1947  store <vscale x 16 x i8> %scalable_arg, ptr %ptr1948  ret void1949}1950 1951define void @sdot_form_4x_tuple(ptr %ptr, i64 %stride) #0 {1952; CHECK-LABEL: sdot_form_4x_tuple:1953; CHECK:       // %bb.0: // %entry1954; CHECK-NEXT:    lsl x9, x1, #11955; CHECK-NEXT:    ptrue pn8.b1956; CHECK-NEXT:    mov w8, wzr1957; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]1958; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]1959; CHECK-NEXT:    add x10, x9, x11960; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]1961; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]1962; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]1963; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]1964; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]1965; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]1966; CHECK-NEXT:    ret1967entry:1968  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()1969  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)1970  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 01971  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 11972  %4 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 21973  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 31974  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride1975  %6 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)1976  %7 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 01977  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 11978  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 21979  %10 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 31980  %mul3 = shl i64 %stride, 11981  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul31982  %11 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)1983  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 01984  %13 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 11985  %14 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 21986  %15 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 31987  %mul5 = mul i64 %stride, 31988  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul51989  %16 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)1990  %17 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 01991  %18 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 11992  %19 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 21993  %20 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 31994  tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %7, <vscale x 16 x i8> %12, <vscale x 16 x i8> %17, <vscale x 16 x i8> poison, i32 0)1995  tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> poison, i32 0)1996  tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> poison, i32 0)1997  tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> poison, i32 0)1998  ret void1999}2000 2001define void @sdot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8> %scalable_arg) #0 {2002; CHECK-LABEL: sdot_form_4x_tuple_svecc:2003; CHECK:       // %bb.0: // %entry2004; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill2005; CHECK-NEXT:    addvl sp, sp, #-92006; CHECK-NEXT:    lsl x9, x1, #12007; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill2008; CHECK-NEXT:    ptrue pn8.b2009; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill2010; CHECK-NEXT:    mov w8, wzr2011; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill2012; CHECK-NEXT:    add x10, x9, x12013; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill2014; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill2015; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill2016; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill2017; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill2018; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill2019; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]2020; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]2021; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]2022; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]2023; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]2024; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]2025; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]2026; CHECK-NEXT:    sdot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]2027; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload2028; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload2029; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload2030; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload2031; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload2032; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload2033; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload2034; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload2035; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload2036; CHECK-NEXT:    str z0, [x0]2037; CHECK-NEXT:    addvl sp, sp, #92038; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload2039; CHECK-NEXT:    ret2040entry:2041  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()2042  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)2043  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 02044  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 12045  %4 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 22046  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 32047  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride2048  %6 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)2049  %7 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 02050  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 12051  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 22052  %10 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 32053  %mul3 = shl i64 %stride, 12054  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul32055  %11 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)2056  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 02057  %13 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 12058  %14 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 22059  %15 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 32060  %mul5 = mul i64 %stride, 32061  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul52062  %16 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)2063  %17 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 02064  %18 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 12065  %19 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 22066  %20 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 32067  tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %7, <vscale x 16 x i8> %12, <vscale x 16 x i8> %17, <vscale x 16 x i8> poison, i32 0)2068  tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> poison, i32 0)2069  tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> poison, i32 0)2070  tail call void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> poison, i32 0)2071  store <vscale x 16 x i8> %scalable_arg, ptr %ptr2072  ret void2073}2074 2075define void @sdot_lane_za64_u16_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2) #1 {2076; CHECK-LABEL: sdot_lane_za64_u16_vg1x2:2077; CHECK:       // %bb.0:2078; CHECK-NEXT:    mov z5.d, z2.d2079; CHECK-NEXT:    mov z4.d, z1.d2080; CHECK-NEXT:    mov w8, w02081; CHECK-NEXT:    sdot za.d[w8, 0, vgx2], { z4.h, z5.h }, z3.h[1]2082; CHECK-NEXT:    sdot za.d[w8, 7, vgx2], { z4.h, z5.h }, z3.h[1]2083; CHECK-NEXT:    ret2084  call void @llvm.aarch64.sme.sdot.lane.za64.vg1x2.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, i32 1)2085  %slice2 = add i32 %slice, 72086  call void @llvm.aarch64.sme.sdot.lane.za64.vg1x2.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, i32 1)2087  ret void2088}2089 2090define void @sdot_lane_za64_u16_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3, <vscale x 8 x i16> %zn4) #1 {2091; CHECK-LABEL: sdot_lane_za64_u16_vg1x4:2092; CHECK:       // %bb.0:2093; CHECK-NEXT:    mov z27.d, z4.d2094; CHECK-NEXT:    mov z26.d, z3.d2095; CHECK-NEXT:    mov w8, w02096; CHECK-NEXT:    mov z25.d, z2.d2097; CHECK-NEXT:    mov z24.d, z1.d2098; CHECK-NEXT:    sdot za.d[w8, 0, vgx4], { z24.h - z27.h }, z5.h[1]2099; CHECK-NEXT:    sdot za.d[w8, 7, vgx4], { z24.h - z27.h }, z5.h[1]2100; CHECK-NEXT:    ret2101  call void @llvm.aarch64.sme.sdot.lane.za64.vg1x4.nxv8i16(i32 %slice, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,2102                                                           <vscale x 8 x i16> %zn4, i32 1)2103  %slice2 = add i32 %slice, 72104  call void @llvm.aarch64.sme.sdot.lane.za64.vg1x4.nxv8i16(i32 %slice2, <vscale x 8 x i16> %zn0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zn3,2105                                                           <vscale x 8 x i16> %zn4, i32 1)2106  ret void2107}2108 2109 2110 2111define void @sudot_lane_za32_u8_vg1x2(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2) #0 {2112; CHECK-LABEL: sudot_lane_za32_u8_vg1x2:2113; CHECK:       // %bb.0:2114; CHECK-NEXT:    mov z5.d, z2.d2115; CHECK-NEXT:    mov z4.d, z1.d2116; CHECK-NEXT:    mov w8, w02117; CHECK-NEXT:    sudot za.s[w8, 0, vgx2], { z4.b, z5.b }, z3.b[3]2118; CHECK-NEXT:    sudot za.s[w8, 7, vgx2], { z4.b, z5.b }, z3.b[3]2119; CHECK-NEXT:    ret2120  call void @llvm.aarch64.sme.sudot.lane.za32.vg1x2.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, i32 3)2121  %slice2 = add i32 %slice, 72122  call void @llvm.aarch64.sme.sudot.lane.za32.vg1x2.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, i32 3)2123  ret void2124}2125 2126define void @sudot_lane_za32_u8_vg1x4(i32 %slice, <vscale x 16 x i8> %unused, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3, <vscale x 16 x i8> %zn4) #0 {2127; CHECK-LABEL: sudot_lane_za32_u8_vg1x4:2128; CHECK:       // %bb.0:2129; CHECK-NEXT:    mov z27.d, z4.d2130; CHECK-NEXT:    mov z26.d, z3.d2131; CHECK-NEXT:    mov w8, w02132; CHECK-NEXT:    mov z25.d, z2.d2133; CHECK-NEXT:    mov z24.d, z1.d2134; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z24.b - z27.b }, z5.b[3]2135; CHECK-NEXT:    sudot za.s[w8, 7, vgx4], { z24.b - z27.b }, z5.b[3]2136; CHECK-NEXT:    ret2137  call void @llvm.aarch64.sme.sudot.lane.za32.vg1x4.nxv16i8(i32 %slice, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,2138                                                            <vscale x 16 x i8> %zn4, i32 3)2139  %slice2 = add i32 %slice, 72140  call void @llvm.aarch64.sme.sudot.lane.za32.vg1x4.nxv16i8(i32 %slice2, <vscale x 16 x i8> %zn0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zn3,2141                                                            <vscale x 16 x i8> %zn4, i32 3)2142  ret void2143}2144 2145define void @sudot_form_2x_tuple(ptr %ptr, i64 %stride) #0 {2146; CHECK-LABEL: sudot_form_2x_tuple:2147; CHECK:       // %bb.0: // %entry2148; CHECK-NEXT:    ptrue pn8.b2149; CHECK-NEXT:    mov w8, wzr2150; CHECK-NEXT:    ld1b { z16.b, z24.b }, pn8/z, [x0]2151; CHECK-NEXT:    ld1b { z17.b, z25.b }, pn8/z, [x0, x1]2152; CHECK-NEXT:    sudot za.s[w8, 0, vgx2], { z16.b, z17.b }, z0.b[0]2153; CHECK-NEXT:    sudot za.s[w8, 0, vgx2], { z24.b, z25.b }, z0.b[0]2154; CHECK-NEXT:    ret2155entry:2156  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()2157  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)2158  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 02159  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 12160  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride2161  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)2162  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 02163  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 12164  tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> poison, i32 0)2165  tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6, <vscale x 16 x i8> poison, i32 0)2166  ret void2167}2168 2169define void @sudot_form_2x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8> %scalable_arg) #0 {2170; CHECK-LABEL: sudot_form_2x_tuple_svecc:2171; CHECK:       // %bb.0: // %entry2172; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill2173; CHECK-NEXT:    addvl sp, sp, #-32174; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill2175; CHECK-NEXT:    ptrue pn8.b2176; CHECK-NEXT:    mov w8, wzr2177; CHECK-NEXT:    str z11, [sp, #1, mul vl] // 16-byte Folded Spill2178; CHECK-NEXT:    str z10, [sp, #2, mul vl] // 16-byte Folded Spill2179; CHECK-NEXT:    ld1b { z2.b, z10.b }, pn8/z, [x0]2180; CHECK-NEXT:    ld1b { z3.b, z11.b }, pn8/z, [x0, x1]2181; CHECK-NEXT:    sudot za.s[w8, 0, vgx2], { z2.b, z3.b }, z0.b[0]2182; CHECK-NEXT:    sudot za.s[w8, 0, vgx2], { z10.b, z11.b }, z0.b[0]2183; CHECK-NEXT:    ldr z11, [sp, #1, mul vl] // 16-byte Folded Reload2184; CHECK-NEXT:    ldr z10, [sp, #2, mul vl] // 16-byte Folded Reload2185; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload2186; CHECK-NEXT:    str z0, [x0]2187; CHECK-NEXT:    addvl sp, sp, #32188; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload2189; CHECK-NEXT:    ret2190entry:2191  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()2192  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)2193  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 02194  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 12195  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride2196  %4 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)2197  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 02198  %6 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %4, 12199  tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %5, <vscale x 16 x i8> poison, i32 0)2200  tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x2.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %6, <vscale x 16 x i8> poison, i32 0)2201  store <vscale x 16 x i8> %scalable_arg, ptr %ptr2202  ret void2203}2204 2205define void @sudot_form_4x_tuple(ptr %ptr, i64 %stride) #0 {2206; CHECK-LABEL: sudot_form_4x_tuple:2207; CHECK:       // %bb.0: // %entry2208; CHECK-NEXT:    lsl x9, x1, #12209; CHECK-NEXT:    ptrue pn8.b2210; CHECK-NEXT:    mov w8, wzr2211; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]2212; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]2213; CHECK-NEXT:    add x10, x9, x12214; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]2215; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]2216; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]2217; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]2218; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]2219; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]2220; CHECK-NEXT:    ret2221entry:2222  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()2223  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)2224  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 02225  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 12226  %4 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 22227  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 32228  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride2229  %6 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)2230  %7 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 02231  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 12232  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 22233  %10 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 32234  %mul3 = shl i64 %stride, 12235  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul32236  %11 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)2237  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 02238  %13 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 12239  %14 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 22240  %15 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 32241  %mul5 = mul i64 %stride, 32242  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul52243  %16 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)2244  %17 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 02245  %18 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 12246  %19 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 22247  %20 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 32248  tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %7, <vscale x 16 x i8> %12, <vscale x 16 x i8> %17, <vscale x 16 x i8> poison, i32 0)2249  tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> poison, i32 0)2250  tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> poison, i32 0)2251  tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> poison, i32 0)2252  ret void2253}2254 2255define void @sudot_form_4x_tuple_svecc(ptr %ptr, i64 %stride, <vscale x 16 x i8> %scalable_arg) #0 {2256; CHECK-LABEL: sudot_form_4x_tuple_svecc:2257; CHECK:       // %bb.0: // %entry2258; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill2259; CHECK-NEXT:    addvl sp, sp, #-92260; CHECK-NEXT:    lsl x9, x1, #12261; CHECK-NEXT:    str p8, [sp, #7, mul vl] // 2-byte Spill2262; CHECK-NEXT:    ptrue pn8.b2263; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill2264; CHECK-NEXT:    mov w8, wzr2265; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill2266; CHECK-NEXT:    add x10, x9, x12267; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill2268; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill2269; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill2270; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill2271; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill2272; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill2273; CHECK-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]2274; CHECK-NEXT:    ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]2275; CHECK-NEXT:    ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]2276; CHECK-NEXT:    ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]2277; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]2278; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]2279; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]2280; CHECK-NEXT:    sudot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]2281; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload2282; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload2283; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload2284; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload2285; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload2286; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload2287; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload2288; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload2289; CHECK-NEXT:    ldr p8, [sp, #7, mul vl] // 2-byte Reload2290; CHECK-NEXT:    str z0, [x0]2291; CHECK-NEXT:    addvl sp, sp, #92292; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload2293; CHECK-NEXT:    ret2294entry:2295  %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()2296  %1 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %ptr)2297  %2 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 02298  %3 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 12299  %4 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 22300  %5 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %1, 32301  %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride2302  %6 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx2)2303  %7 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 02304  %8 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 12305  %9 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 22306  %10 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %6, 32307  %mul3 = shl i64 %stride, 12308  %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul32309  %11 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx4)2310  %12 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 02311  %13 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 12312  %14 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 22313  %15 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %11, 32314  %mul5 = mul i64 %stride, 32315  %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul52316  %16 = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %0, ptr %arrayidx6)2317  %17 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 02318  %18 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 12319  %19 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 22320  %20 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %16, 32321  tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %2, <vscale x 16 x i8> %7, <vscale x 16 x i8> %12, <vscale x 16 x i8> %17, <vscale x 16 x i8> poison, i32 0)2322  tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %3, <vscale x 16 x i8> %8, <vscale x 16 x i8> %13, <vscale x 16 x i8> %18, <vscale x 16 x i8> poison, i32 0)2323  tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %4, <vscale x 16 x i8> %9, <vscale x 16 x i8> %14, <vscale x 16 x i8> %19, <vscale x 16 x i8> poison, i32 0)2324  tail call void @llvm.aarch64.sme.sudot.lane.za32.vg1x4.nxv16i8(i32 0, <vscale x 16 x i8> %5, <vscale x 16 x i8> %10, <vscale x 16 x i8> %15, <vscale x 16 x i8> %20, <vscale x 16 x i8> poison, i32 0)2325  store <vscale x 16 x i8> %scalable_arg, ptr %ptr2326  ret void2327}2328 2329 2330attributes #0 = { nounwind "target-features"="+sme2" }2331attributes #1 = { nounwind "target-features"="+sme2,+sme-i16i64" }2332 2333; == Multi, multi (unsigned)2334 2335declare void @llvm.aarch64.sme.udot.za32.vg1x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)2336declare void @llvm.aarch64.sme.udot.za32.vg1x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>,2337                                                       <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)2338declare void @llvm.aarch64.sme.udot.za32.vg1x2.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)2339declare void @llvm.aarch64.sme.udot.za32.vg1x4.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>,2340                                                       <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)2341declare void @llvm.aarch64.sme.udot.za64.vg1x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)2342declare void @llvm.aarch64.sme.udot.za64.vg1x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>,2343                                                       <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)2344declare void @llvm.aarch64.sme.usdot.za32.vg1x2.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)2345declare void @llvm.aarch64.sme.usdot.za32.vg1x4.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>,2346                                                        <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)2347 2348; == Multi, multi (signed)2349 2350declare void @llvm.aarch64.sme.sdot.za32.vg1x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)2351declare void @llvm.aarch64.sme.sdot.za32.vg1x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>,2352                                                       <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)2353declare void @llvm.aarch64.sme.sdot.za32.vg1x2.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)2354declare void @llvm.aarch64.sme.sdot.za32.vg1x4.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>,2355                                                       <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)2356declare void @llvm.aarch64.sme.sdot.za64.vg1x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)2357declare void @llvm.aarch64.sme.sdot.za64.vg1x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>,2358                                                       <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)2359 2360; == Multi, single (unsigned)2361 2362declare void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)2363declare void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)2364declare void @llvm.aarch64.sme.udot.single.za32.vg1x2.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)2365declare void @llvm.aarch64.sme.udot.single.za32.vg1x4.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)2366declare void @llvm.aarch64.sme.udot.single.za64.vg1x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)2367declare void @llvm.aarch64.sme.udot.single.za64.vg1x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)2368declare void @llvm.aarch64.sme.usdot.single.za32.vg1x2.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)2369declare void @llvm.aarch64.sme.usdot.single.za32.vg1x4.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)2370 2371; == Multi, single (signed)2372 2373declare void @llvm.aarch64.sme.sdot.single.za32.vg1x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)2374declare void @llvm.aarch64.sme.sdot.single.za32.vg1x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)2375declare void @llvm.aarch64.sme.sdot.single.za32.vg1x2.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)2376declare void @llvm.aarch64.sme.sdot.single.za32.vg1x4.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)2377declare void @llvm.aarch64.sme.sdot.single.za64.vg1x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)2378declare void @llvm.aarch64.sme.sdot.single.za64.vg1x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)2379declare void @llvm.aarch64.sme.sudot.single.za32.vg1x2.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)2380declare void @llvm.aarch64.sme.sudot.single.za32.vg1x4.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)2381 2382; == Multi, indexed (unsigned)2383 2384declare void @llvm.aarch64.sme.udot.lane.za32.vg1x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)2385declare void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)2386declare void @llvm.aarch64.sme.udot.lane.za32.vg1x2.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, i32)2387declare void @llvm.aarch64.sme.udot.lane.za32.vg1x4.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, i32)2388declare void @llvm.aarch64.sme.udot.lane.za64.vg1x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)2389declare void @llvm.aarch64.sme.udot.lane.za64.vg1x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)2390declare void @llvm.aarch64.sme.usdot.lane.za32.vg1x2.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, i32)2391declare void @llvm.aarch64.sme.usdot.lane.za32.vg1x4.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, i32)2392 2393; == Multi, indexed (signed)2394 2395declare void @llvm.aarch64.sme.sdot.lane.za32.vg1x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)2396declare void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)2397declare void @llvm.aarch64.sme.sdot.lane.za32.vg1x2.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, i32)2398declare void @llvm.aarch64.sme.sdot.lane.za32.vg1x4.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, i32)2399declare void @llvm.aarch64.sme.sdot.lane.za64.vg1x2.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)2400declare void @llvm.aarch64.sme.sdot.lane.za64.vg1x4.nxv8i16(i32, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, i32)2401declare void @llvm.aarch64.sme.sudot.lane.za32.vg1x2.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, i32)2402declare void @llvm.aarch64.sme.sudot.lane.za32.vg1x4.nxv16i8(i32, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, i32)2403