brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod,+i8mm    < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD3; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod,+i8mm -global-isel < %s | FileCheck %s --check-prefixes=CHECK,CHECK-GI4 5declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)6declare i32 @llvm.vector.reduce.add.v5i32(<5 x i32>)7declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)8declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)9declare i32 @llvm.vector.reduce.add.v24i32(<24 x i32>)10declare i32 @llvm.vector.reduce.add.v25i32(<25 x i32>)11declare i32 @llvm.vector.reduce.add.v32i32(<32 x i32>)12declare i32 @llvm.vector.reduce.add.v33i32(<33 x i32>)13declare i32 @llvm.vector.reduce.add.v48i32(<48 x i32>)14declare i32 @llvm.vector.reduce.add.v64i32(<64 x i32>)15 16define i32 @test_udot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {17; CHECK-SD-LABEL: test_udot_v4i8:18; CHECK-SD:       // %bb.0: // %entry19; CHECK-SD-NEXT:    ldr s0, [x0]20; CHECK-SD-NEXT:    ldr s1, [x1]21; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #022; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #023; CHECK-SD-NEXT:    umull v0.4s, v1.4h, v0.4h24; CHECK-SD-NEXT:    addv s0, v0.4s25; CHECK-SD-NEXT:    fmov w8, s026; CHECK-SD-NEXT:    add w0, w8, w227; CHECK-SD-NEXT:    ret28;29; CHECK-GI-LABEL: test_udot_v4i8:30; CHECK-GI:       // %bb.0: // %entry31; CHECK-GI-NEXT:    ldr w8, [x1]32; CHECK-GI-NEXT:    ldr w9, [x0]33; CHECK-GI-NEXT:    fmov s0, w834; CHECK-GI-NEXT:    fmov s1, w935; CHECK-GI-NEXT:    mov b2, v0.b[1]36; CHECK-GI-NEXT:    mov v3.b[0], v0.b[0]37; CHECK-GI-NEXT:    mov b4, v1.b[1]38; CHECK-GI-NEXT:    mov v5.b[0], v1.b[0]39; CHECK-GI-NEXT:    mov v3.b[1], v2.b[0]40; CHECK-GI-NEXT:    mov b2, v0.b[2]41; CHECK-GI-NEXT:    mov b0, v0.b[3]42; CHECK-GI-NEXT:    mov v5.b[1], v4.b[0]43; CHECK-GI-NEXT:    mov b4, v1.b[2]44; CHECK-GI-NEXT:    mov b1, v1.b[3]45; CHECK-GI-NEXT:    mov v3.b[2], v2.b[0]46; CHECK-GI-NEXT:    mov v5.b[2], v4.b[0]47; CHECK-GI-NEXT:    mov v3.b[3], v0.b[0]48; CHECK-GI-NEXT:    mov v5.b[3], v1.b[0]49; CHECK-GI-NEXT:    ushll v0.8h, v3.8b, #050; CHECK-GI-NEXT:    ushll v1.8h, v5.8b, #051; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h52; CHECK-GI-NEXT:    uaddlv s0, v0.4h53; CHECK-GI-NEXT:    fmov w8, s054; CHECK-GI-NEXT:    add w0, w8, w255; CHECK-GI-NEXT:    ret56entry:57  %0 = load <4 x i8>, ptr %a58  %1 = zext <4 x i8> %0 to <4 x i32>59  %2 = load <4 x i8>, ptr %b60  %3 = zext <4 x i8> %2 to <4 x i32>61  %4 = mul nuw nsw <4 x i32> %3, %162  %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)63  %op.extra = add i32 %5, %sum64  ret i32 %op.extra65}66 67define i32 @test_udot_v4i8_nomla(ptr nocapture readonly %a1) {68; CHECK-SD-LABEL: test_udot_v4i8_nomla:69; CHECK-SD:       // %bb.0: // %entry70; CHECK-SD-NEXT:    ldr s0, [x0]71; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #072; CHECK-SD-NEXT:    uaddlv s0, v0.4h73; CHECK-SD-NEXT:    fmov w0, s074; CHECK-SD-NEXT:    ret75;76; CHECK-GI-LABEL: test_udot_v4i8_nomla:77; CHECK-GI:       // %bb.0: // %entry78; CHECK-GI-NEXT:    ldr w8, [x0]79; CHECK-GI-NEXT:    fmov s0, w880; CHECK-GI-NEXT:    mov b1, v0.b[1]81; CHECK-GI-NEXT:    mov v2.b[0], v0.b[0]82; CHECK-GI-NEXT:    mov b3, v0.b[2]83; CHECK-GI-NEXT:    mov b0, v0.b[3]84; CHECK-GI-NEXT:    mov v2.b[1], v1.b[0]85; CHECK-GI-NEXT:    mov v2.b[2], v3.b[0]86; CHECK-GI-NEXT:    mov v2.b[3], v0.b[0]87; CHECK-GI-NEXT:    ushll v0.8h, v2.8b, #088; CHECK-GI-NEXT:    uaddlv s0, v0.4h89; CHECK-GI-NEXT:    fmov w8, s090; CHECK-GI-NEXT:    and w0, w8, #0xffff91; CHECK-GI-NEXT:    ret92entry:93  %0 = load <4 x i8>, ptr %a194  %1 = zext <4 x i8> %0 to <4 x i32>95  %2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %1)96  ret i32 %297}98define i32 @test_sdot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {99; CHECK-SD-LABEL: test_sdot_v4i8:100; CHECK-SD:       // %bb.0: // %entry101; CHECK-SD-NEXT:    ldr s0, [x0]102; CHECK-SD-NEXT:    ldr s1, [x1]103; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0104; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0105; CHECK-SD-NEXT:    smull v0.4s, v1.4h, v0.4h106; CHECK-SD-NEXT:    addv s0, v0.4s107; CHECK-SD-NEXT:    fmov w8, s0108; CHECK-SD-NEXT:    add w0, w8, w2109; CHECK-SD-NEXT:    ret110;111; CHECK-GI-LABEL: test_sdot_v4i8:112; CHECK-GI:       // %bb.0: // %entry113; CHECK-GI-NEXT:    ldr w8, [x1]114; CHECK-GI-NEXT:    ldr w9, [x0]115; CHECK-GI-NEXT:    fmov s0, w8116; CHECK-GI-NEXT:    fmov s1, w9117; CHECK-GI-NEXT:    mov b2, v0.b[1]118; CHECK-GI-NEXT:    mov v3.b[0], v0.b[0]119; CHECK-GI-NEXT:    mov b4, v1.b[1]120; CHECK-GI-NEXT:    mov v5.b[0], v1.b[0]121; CHECK-GI-NEXT:    mov v3.b[1], v2.b[0]122; CHECK-GI-NEXT:    mov b2, v0.b[2]123; CHECK-GI-NEXT:    mov b0, v0.b[3]124; CHECK-GI-NEXT:    mov v5.b[1], v4.b[0]125; CHECK-GI-NEXT:    mov b4, v1.b[2]126; CHECK-GI-NEXT:    mov b1, v1.b[3]127; CHECK-GI-NEXT:    mov v3.b[2], v2.b[0]128; CHECK-GI-NEXT:    mov v5.b[2], v4.b[0]129; CHECK-GI-NEXT:    mov v3.b[3], v0.b[0]130; CHECK-GI-NEXT:    mov v5.b[3], v1.b[0]131; CHECK-GI-NEXT:    sshll v0.8h, v3.8b, #0132; CHECK-GI-NEXT:    sshll v1.8h, v5.8b, #0133; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h134; CHECK-GI-NEXT:    saddlv s0, v0.4h135; CHECK-GI-NEXT:    fmov w8, s0136; CHECK-GI-NEXT:    add w0, w8, w2137; CHECK-GI-NEXT:    ret138entry:139  %0 = load <4 x i8>, ptr %a140  %1 = sext <4 x i8> %0 to <4 x i32>141  %2 = load <4 x i8>, ptr %b142  %3 = sext <4 x i8> %2 to <4 x i32>143  %4 = mul nsw <4 x i32> %3, %1144  %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)145  %op.extra = add nsw i32 %5, %sum146  ret i32 %op.extra147}148 149define i32 @test_sdot_v4i8_double(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {150; CHECK-SD-LABEL: test_sdot_v4i8_double:151; CHECK-SD:       // %bb.0: // %entry152; CHECK-SD-NEXT:    ushll v3.4s, v3.4h, #0153; CHECK-SD-NEXT:    ushll v2.4s, v2.4h, #0154; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #0155; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0156; CHECK-SD-NEXT:    shl v2.4s, v2.4s, #24157; CHECK-SD-NEXT:    shl v3.4s, v3.4s, #24158; CHECK-SD-NEXT:    shl v1.4s, v1.4s, #24159; CHECK-SD-NEXT:    shl v0.4s, v0.4s, #24160; CHECK-SD-NEXT:    sshr v2.4s, v2.4s, #24161; CHECK-SD-NEXT:    sshr v3.4s, v3.4s, #24162; CHECK-SD-NEXT:    sshr v1.4s, v1.4s, #24163; CHECK-SD-NEXT:    sshr v0.4s, v0.4s, #24164; CHECK-SD-NEXT:    mul v2.4s, v2.4s, v3.4s165; CHECK-SD-NEXT:    mla v2.4s, v0.4s, v1.4s166; CHECK-SD-NEXT:    addv s0, v2.4s167; CHECK-SD-NEXT:    fmov w0, s0168; CHECK-SD-NEXT:    ret169;170; CHECK-GI-LABEL: test_sdot_v4i8_double:171; CHECK-GI:       // %bb.0: // %entry172; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #8173; CHECK-GI-NEXT:    shl v1.4h, v1.4h, #8174; CHECK-GI-NEXT:    shl v2.4h, v2.4h, #8175; CHECK-GI-NEXT:    shl v3.4h, v3.4h, #8176; CHECK-GI-NEXT:    sshr v0.4h, v0.4h, #8177; CHECK-GI-NEXT:    sshr v1.4h, v1.4h, #8178; CHECK-GI-NEXT:    sshr v2.4h, v2.4h, #8179; CHECK-GI-NEXT:    sshr v3.4h, v3.4h, #8180; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h181; CHECK-GI-NEXT:    mul v1.4h, v2.4h, v3.4h182; CHECK-GI-NEXT:    saddlv s0, v0.4h183; CHECK-GI-NEXT:    saddlv s1, v1.4h184; CHECK-GI-NEXT:    fmov w8, s0185; CHECK-GI-NEXT:    fmov w9, s1186; CHECK-GI-NEXT:    add w0, w8, w9187; CHECK-GI-NEXT:    ret188entry:189  %az = sext <4 x i8> %a to <4 x i32>190  %bz = sext <4 x i8> %b to <4 x i32>191  %m1 = mul nuw nsw <4 x i32> %az, %bz192  %r1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m1)193  %cz = sext <4 x i8> %c to <4 x i32>194  %dz = sext <4 x i8> %d to <4 x i32>195  %m2 = mul nuw nsw <4 x i32> %cz, %dz196  %r2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m2)197  %x = add i32 %r1, %r2198  ret i32 %x199}200 201define i32 @test_sdot_v4i8_double_nomla(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {202; CHECK-SD-LABEL: test_sdot_v4i8_double_nomla:203; CHECK-SD:       // %bb.0: // %entry204; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0205; CHECK-SD-NEXT:    ushll v1.4s, v2.4h, #0206; CHECK-SD-NEXT:    shl v0.4s, v0.4s, #24207; CHECK-SD-NEXT:    shl v1.4s, v1.4s, #24208; CHECK-SD-NEXT:    sshr v0.4s, v0.4s, #24209; CHECK-SD-NEXT:    ssra v0.4s, v1.4s, #24210; CHECK-SD-NEXT:    addv s0, v0.4s211; CHECK-SD-NEXT:    fmov w0, s0212; CHECK-SD-NEXT:    ret213;214; CHECK-GI-LABEL: test_sdot_v4i8_double_nomla:215; CHECK-GI:       // %bb.0: // %entry216; CHECK-GI-NEXT:    shl v1.4h, v2.4h, #8217; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #8218; CHECK-GI-NEXT:    sshr v1.4h, v1.4h, #8219; CHECK-GI-NEXT:    sshr v0.4h, v0.4h, #8220; CHECK-GI-NEXT:    saddlv s1, v1.4h221; CHECK-GI-NEXT:    saddlv s0, v0.4h222; CHECK-GI-NEXT:    fmov w8, s1223; CHECK-GI-NEXT:    fmov w9, s0224; CHECK-GI-NEXT:    sxth w8, w8225; CHECK-GI-NEXT:    add w0, w8, w9, sxth226; CHECK-GI-NEXT:    ret227entry:228  %az = sext <4 x i8> %a to <4 x i32>229  %r1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %az)230  %cz = sext <4 x i8> %c to <4 x i32>231  %r2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %cz)232  %x = add i32 %r1, %r2233  ret i32 %x234}235 236define i32 @test_usdot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {237; CHECK-SD-LABEL: test_usdot_v4i8:238; CHECK-SD:       // %bb.0: // %entry239; CHECK-SD-NEXT:    ldr s0, [x0]240; CHECK-SD-NEXT:    ldr s1, [x1]241; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0242; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #0243; CHECK-SD-NEXT:    smull v0.4s, v1.4h, v0.4h244; CHECK-SD-NEXT:    addv s0, v0.4s245; CHECK-SD-NEXT:    fmov w8, s0246; CHECK-SD-NEXT:    add w0, w8, w2247; CHECK-SD-NEXT:    ret248;249; CHECK-GI-LABEL: test_usdot_v4i8:250; CHECK-GI:       // %bb.0: // %entry251; CHECK-GI-NEXT:    ldr w8, [x0]252; CHECK-GI-NEXT:    ldr w9, [x1]253; CHECK-GI-NEXT:    fmov s0, w8254; CHECK-GI-NEXT:    fmov s2, w9255; CHECK-GI-NEXT:    uxtb w8, w8256; CHECK-GI-NEXT:    sxtb w9, w9257; CHECK-GI-NEXT:    mov b1, v0.b[1]258; CHECK-GI-NEXT:    mov b3, v0.b[2]259; CHECK-GI-NEXT:    mov b5, v2.b[2]260; CHECK-GI-NEXT:    mov b4, v0.b[3]261; CHECK-GI-NEXT:    mov b0, v2.b[1]262; CHECK-GI-NEXT:    mov b6, v2.b[3]263; CHECK-GI-NEXT:    fmov s2, w9264; CHECK-GI-NEXT:    fmov w10, s1265; CHECK-GI-NEXT:    fmov w11, s3266; CHECK-GI-NEXT:    fmov s1, w8267; CHECK-GI-NEXT:    fmov w13, s5268; CHECK-GI-NEXT:    fmov w8, s4269; CHECK-GI-NEXT:    fmov w12, s0270; CHECK-GI-NEXT:    uxtb w10, w10271; CHECK-GI-NEXT:    uxtb w11, w11272; CHECK-GI-NEXT:    sxtb w13, w13273; CHECK-GI-NEXT:    uxtb w8, w8274; CHECK-GI-NEXT:    sxtb w12, w12275; CHECK-GI-NEXT:    mov v1.h[1], w10276; CHECK-GI-NEXT:    fmov w10, s6277; CHECK-GI-NEXT:    fmov s0, w11278; CHECK-GI-NEXT:    fmov s3, w13279; CHECK-GI-NEXT:    mov v2.h[1], w12280; CHECK-GI-NEXT:    sxtb w10, w10281; CHECK-GI-NEXT:    mov v0.h[1], w8282; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0283; CHECK-GI-NEXT:    mov v3.h[1], w10284; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0285; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0286; CHECK-GI-NEXT:    sshll v3.4s, v3.4h, #0287; CHECK-GI-NEXT:    mov v1.d[1], v0.d[0]288; CHECK-GI-NEXT:    mov v2.d[1], v3.d[0]289; CHECK-GI-NEXT:    mul v0.4s, v2.4s, v1.4s290; CHECK-GI-NEXT:    addv s0, v0.4s291; CHECK-GI-NEXT:    fmov w8, s0292; CHECK-GI-NEXT:    add w0, w8, w2293; CHECK-GI-NEXT:    ret294entry:295  %0 = load <4 x i8>, ptr %a296  %1 = zext <4 x i8> %0 to <4 x i32>297  %2 = load <4 x i8>, ptr %b298  %3 = sext <4 x i8> %2 to <4 x i32>299  %4 = mul nsw <4 x i32> %3, %1300  %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)301  %op.extra = add nsw i32 %5, %sum302  ret i32 %op.extra303}304 305define i32 @test_usdot_v4i8_double(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) {306; CHECK-SD-LABEL: test_usdot_v4i8_double:307; CHECK-SD:       // %bb.0: // %entry308; CHECK-SD-NEXT:    ushll v3.4s, v3.4h, #0309; CHECK-SD-NEXT:    bic v2.4h, #255, lsl #8310; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #0311; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8312; CHECK-SD-NEXT:    shl v3.4s, v3.4s, #24313; CHECK-SD-NEXT:    ushll v2.4s, v2.4h, #0314; CHECK-SD-NEXT:    shl v1.4s, v1.4s, #24315; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0316; CHECK-SD-NEXT:    sshr v3.4s, v3.4s, #24317; CHECK-SD-NEXT:    sshr v1.4s, v1.4s, #24318; CHECK-SD-NEXT:    mul v2.4s, v2.4s, v3.4s319; CHECK-SD-NEXT:    mla v2.4s, v0.4s, v1.4s320; CHECK-SD-NEXT:    addv s0, v2.4s321; CHECK-SD-NEXT:    fmov w0, s0322; CHECK-SD-NEXT:    ret323;324; CHECK-GI-LABEL: test_usdot_v4i8_double:325; CHECK-GI:       // %bb.0: // %entry326; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0327; CHECK-GI-NEXT:    ushll v3.4s, v3.4h, #0328; CHECK-GI-NEXT:    movi v4.2d, #0x0000ff000000ff329; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0330; CHECK-GI-NEXT:    ushll v2.4s, v2.4h, #0331; CHECK-GI-NEXT:    shl v1.4s, v1.4s, #24332; CHECK-GI-NEXT:    shl v3.4s, v3.4s, #24333; CHECK-GI-NEXT:    and v0.16b, v0.16b, v4.16b334; CHECK-GI-NEXT:    and v2.16b, v2.16b, v4.16b335; CHECK-GI-NEXT:    sshr v1.4s, v1.4s, #24336; CHECK-GI-NEXT:    sshr v3.4s, v3.4s, #24337; CHECK-GI-NEXT:    mul v0.4s, v0.4s, v1.4s338; CHECK-GI-NEXT:    mul v1.4s, v2.4s, v3.4s339; CHECK-GI-NEXT:    addv s0, v0.4s340; CHECK-GI-NEXT:    addv s1, v1.4s341; CHECK-GI-NEXT:    fmov w8, s0342; CHECK-GI-NEXT:    fmov w9, s1343; CHECK-GI-NEXT:    add w0, w8, w9344; CHECK-GI-NEXT:    ret345entry:346  %az = zext <4 x i8> %a to <4 x i32>347  %bz = sext <4 x i8> %b to <4 x i32>348  %m1 = mul nuw nsw <4 x i32> %az, %bz349  %r1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m1)350  %cz = zext <4 x i8> %c to <4 x i32>351  %dz = sext <4 x i8> %d to <4 x i32>352  %m2 = mul nuw nsw <4 x i32> %cz, %dz353  %r2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m2)354  %x = add i32 %r1, %r2355  ret i32 %x356}357 358define i32 @test_udot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {359; CHECK-SD-LABEL: test_udot_v5i8:360; CHECK-SD:       // %bb.0: // %entry361; CHECK-SD-NEXT:    ldr d0, [x0]362; CHECK-SD-NEXT:    ldr d1, [x1]363; CHECK-SD-NEXT:    umull v0.8h, v1.8b, v0.8b364; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000365; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0366; CHECK-SD-NEXT:    mov v1.s[0], v2.s[0]367; CHECK-SD-NEXT:    uaddw v0.4s, v1.4s, v0.4h368; CHECK-SD-NEXT:    addv s0, v0.4s369; CHECK-SD-NEXT:    fmov w8, s0370; CHECK-SD-NEXT:    add w0, w8, w2371; CHECK-SD-NEXT:    ret372;373; CHECK-GI-LABEL: test_udot_v5i8:374; CHECK-GI:       // %bb.0: // %entry375; CHECK-GI-NEXT:    ldr d0, [x0]376; CHECK-GI-NEXT:    ldr d1, [x1]377; CHECK-GI-NEXT:    mov b2, v0.b[1]378; CHECK-GI-NEXT:    mov b3, v1.b[1]379; CHECK-GI-NEXT:    fmov w8, s1380; CHECK-GI-NEXT:    fmov w9, s0381; CHECK-GI-NEXT:    mov b4, v1.b[2]382; CHECK-GI-NEXT:    mov b5, v0.b[2]383; CHECK-GI-NEXT:    mov b6, v0.b[3]384; CHECK-GI-NEXT:    mov b7, v1.b[3]385; CHECK-GI-NEXT:    mov b0, v0.b[4]386; CHECK-GI-NEXT:    uxtb w8, w8387; CHECK-GI-NEXT:    mov b1, v1.b[4]388; CHECK-GI-NEXT:    fmov w10, s3389; CHECK-GI-NEXT:    uxtb w9, w9390; CHECK-GI-NEXT:    fmov w11, s2391; CHECK-GI-NEXT:    fmov s2, w8392; CHECK-GI-NEXT:    fmov w8, s4393; CHECK-GI-NEXT:    fmov s3, w9394; CHECK-GI-NEXT:    fmov w9, s5395; CHECK-GI-NEXT:    uxtb w10, w10396; CHECK-GI-NEXT:    uxtb w11, w11397; CHECK-GI-NEXT:    uxtb w8, w8398; CHECK-GI-NEXT:    mov v2.h[1], w10399; CHECK-GI-NEXT:    mov v3.h[1], w11400; CHECK-GI-NEXT:    uxtb w9, w9401; CHECK-GI-NEXT:    mov v2.h[2], w8402; CHECK-GI-NEXT:    mov v3.h[2], w9403; CHECK-GI-NEXT:    fmov w8, s7404; CHECK-GI-NEXT:    fmov w9, s6405; CHECK-GI-NEXT:    uxtb w8, w8406; CHECK-GI-NEXT:    uxtb w9, w9407; CHECK-GI-NEXT:    mov v2.h[3], w8408; CHECK-GI-NEXT:    fmov w8, s1409; CHECK-GI-NEXT:    mov v3.h[3], w9410; CHECK-GI-NEXT:    fmov w9, s0411; CHECK-GI-NEXT:    uxtb w8, w8412; CHECK-GI-NEXT:    uxtb w9, w9413; CHECK-GI-NEXT:    mov v2.h[4], w8414; CHECK-GI-NEXT:    mov v3.h[4], w9415; CHECK-GI-NEXT:    mul v0.8h, v2.8h, v3.8h416; CHECK-GI-NEXT:    umov w8, v0.h[0]417; CHECK-GI-NEXT:    umov w9, v0.h[4]418; CHECK-GI-NEXT:    umov w10, v0.h[1]419; CHECK-GI-NEXT:    fmov s1, w8420; CHECK-GI-NEXT:    fmov s2, w9421; CHECK-GI-NEXT:    umov w8, v0.h[2]422; CHECK-GI-NEXT:    umov w9, v0.h[3]423; CHECK-GI-NEXT:    mov v1.s[1], w10424; CHECK-GI-NEXT:    mov v2.s[1], wzr425; CHECK-GI-NEXT:    mov v1.s[2], w8426; CHECK-GI-NEXT:    mov v2.s[2], wzr427; CHECK-GI-NEXT:    mov v1.s[3], w9428; CHECK-GI-NEXT:    mov v2.s[3], wzr429; CHECK-GI-NEXT:    add v0.4s, v1.4s, v2.4s430; CHECK-GI-NEXT:    addv s0, v0.4s431; CHECK-GI-NEXT:    fmov w8, s0432; CHECK-GI-NEXT:    add w0, w8, w2433; CHECK-GI-NEXT:    ret434entry:435  %0 = load <5 x i8>, ptr %a436  %1 = zext <5 x i8> %0 to <5 x i32>437  %2 = load <5 x i8>, ptr %b438  %3 = zext <5 x i8> %2 to <5 x i32>439  %4 = mul nuw nsw <5 x i32> %3, %1440  %5 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %4)441  %op.extra = add i32 %5, %sum442  ret i32 %op.extra443}444 445define i32 @test_udot_v5i8_nomla(ptr nocapture readonly %a1) {446; CHECK-SD-LABEL: test_udot_v5i8_nomla:447; CHECK-SD:       // %bb.0: // %entry448; CHECK-SD-NEXT:    ldr d0, [x0]449; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000450; CHECK-SD-NEXT:    ushll v0.8h, v0.8b, #0451; CHECK-SD-NEXT:    ushll2 v2.4s, v0.8h, #0452; CHECK-SD-NEXT:    mov v1.s[0], v2.s[0]453; CHECK-SD-NEXT:    uaddw v0.4s, v1.4s, v0.4h454; CHECK-SD-NEXT:    addv s0, v0.4s455; CHECK-SD-NEXT:    fmov w0, s0456; CHECK-SD-NEXT:    ret457;458; CHECK-GI-LABEL: test_udot_v5i8_nomla:459; CHECK-GI:       // %bb.0: // %entry460; CHECK-GI-NEXT:    ldr d0, [x0]461; CHECK-GI-NEXT:    umov w8, v0.b[0]462; CHECK-GI-NEXT:    umov w9, v0.b[4]463; CHECK-GI-NEXT:    umov w10, v0.b[1]464; CHECK-GI-NEXT:    fmov s1, w8465; CHECK-GI-NEXT:    fmov s2, w9466; CHECK-GI-NEXT:    umov w8, v0.b[2]467; CHECK-GI-NEXT:    umov w9, v0.b[3]468; CHECK-GI-NEXT:    mov v1.s[1], w10469; CHECK-GI-NEXT:    mov v2.s[1], wzr470; CHECK-GI-NEXT:    mov v1.s[2], w8471; CHECK-GI-NEXT:    mov v2.s[2], wzr472; CHECK-GI-NEXT:    mov v1.s[3], w9473; CHECK-GI-NEXT:    mov v2.s[3], wzr474; CHECK-GI-NEXT:    add v0.4s, v1.4s, v2.4s475; CHECK-GI-NEXT:    addv s0, v0.4s476; CHECK-GI-NEXT:    fmov w0, s0477; CHECK-GI-NEXT:    ret478entry:479  %0 = load <5 x i8>, ptr %a1480  %1 = zext <5 x i8> %0 to <5 x i32>481  %2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %1)482  ret i32 %2483}484define i32 @test_sdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {485; CHECK-SD-LABEL: test_sdot_v5i8:486; CHECK-SD:       // %bb.0: // %entry487; CHECK-SD-NEXT:    ldr d0, [x0]488; CHECK-SD-NEXT:    ldr d1, [x1]489; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b490; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000491; CHECK-SD-NEXT:    sshll2 v2.4s, v0.8h, #0492; CHECK-SD-NEXT:    mov v1.s[0], v2.s[0]493; CHECK-SD-NEXT:    saddw v0.4s, v1.4s, v0.4h494; CHECK-SD-NEXT:    addv s0, v0.4s495; CHECK-SD-NEXT:    fmov w8, s0496; CHECK-SD-NEXT:    add w0, w8, w2497; CHECK-SD-NEXT:    ret498;499; CHECK-GI-LABEL: test_sdot_v5i8:500; CHECK-GI:       // %bb.0: // %entry501; CHECK-GI-NEXT:    ldr d0, [x0]502; CHECK-GI-NEXT:    ldr d1, [x1]503; CHECK-GI-NEXT:    mov b2, v0.b[1]504; CHECK-GI-NEXT:    mov b3, v1.b[1]505; CHECK-GI-NEXT:    fmov w8, s1506; CHECK-GI-NEXT:    fmov w9, s0507; CHECK-GI-NEXT:    mov b4, v1.b[2]508; CHECK-GI-NEXT:    mov b5, v0.b[2]509; CHECK-GI-NEXT:    mov b6, v0.b[3]510; CHECK-GI-NEXT:    mov b7, v1.b[3]511; CHECK-GI-NEXT:    mov b0, v0.b[4]512; CHECK-GI-NEXT:    sxtb w8, w8513; CHECK-GI-NEXT:    mov b1, v1.b[4]514; CHECK-GI-NEXT:    fmov w10, s3515; CHECK-GI-NEXT:    sxtb w9, w9516; CHECK-GI-NEXT:    fmov w11, s2517; CHECK-GI-NEXT:    fmov s2, w8518; CHECK-GI-NEXT:    fmov w8, s4519; CHECK-GI-NEXT:    fmov s3, w9520; CHECK-GI-NEXT:    fmov w9, s5521; CHECK-GI-NEXT:    sxtb w10, w10522; CHECK-GI-NEXT:    sxtb w11, w11523; CHECK-GI-NEXT:    sxtb w8, w8524; CHECK-GI-NEXT:    mov v2.h[1], w10525; CHECK-GI-NEXT:    mov v3.h[1], w11526; CHECK-GI-NEXT:    sxtb w9, w9527; CHECK-GI-NEXT:    mov v2.h[2], w8528; CHECK-GI-NEXT:    mov v3.h[2], w9529; CHECK-GI-NEXT:    fmov w8, s7530; CHECK-GI-NEXT:    fmov w9, s6531; CHECK-GI-NEXT:    sxtb w8, w8532; CHECK-GI-NEXT:    sxtb w9, w9533; CHECK-GI-NEXT:    mov v2.h[3], w8534; CHECK-GI-NEXT:    fmov w8, s1535; CHECK-GI-NEXT:    mov v3.h[3], w9536; CHECK-GI-NEXT:    fmov w9, s0537; CHECK-GI-NEXT:    sxtb w8, w8538; CHECK-GI-NEXT:    sxtb w9, w9539; CHECK-GI-NEXT:    mov v2.h[4], w8540; CHECK-GI-NEXT:    mov v3.h[4], w9541; CHECK-GI-NEXT:    mul v0.8h, v2.8h, v3.8h542; CHECK-GI-NEXT:    smov w8, v0.h[0]543; CHECK-GI-NEXT:    smov w9, v0.h[4]544; CHECK-GI-NEXT:    smov w10, v0.h[1]545; CHECK-GI-NEXT:    fmov s1, w8546; CHECK-GI-NEXT:    fmov s2, w9547; CHECK-GI-NEXT:    smov w8, v0.h[2]548; CHECK-GI-NEXT:    smov w9, v0.h[3]549; CHECK-GI-NEXT:    mov v1.s[1], w10550; CHECK-GI-NEXT:    mov v2.s[1], wzr551; CHECK-GI-NEXT:    mov v1.s[2], w8552; CHECK-GI-NEXT:    mov v2.s[2], wzr553; CHECK-GI-NEXT:    mov v1.s[3], w9554; CHECK-GI-NEXT:    mov v2.s[3], wzr555; CHECK-GI-NEXT:    add v0.4s, v1.4s, v2.4s556; CHECK-GI-NEXT:    addv s0, v0.4s557; CHECK-GI-NEXT:    fmov w8, s0558; CHECK-GI-NEXT:    add w0, w8, w2559; CHECK-GI-NEXT:    ret560entry:561  %0 = load <5 x i8>, ptr %a562  %1 = sext <5 x i8> %0 to <5 x i32>563  %2 = load <5 x i8>, ptr %b564  %3 = sext <5 x i8> %2 to <5 x i32>565  %4 = mul nsw <5 x i32> %3, %1566  %5 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %4)567  %op.extra = add nsw i32 %5, %sum568  ret i32 %op.extra569}570 571define i32 @test_sdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) {572; CHECK-SD-LABEL: test_sdot_v5i8_double:573; CHECK-SD:       // %bb.0: // %entry574; CHECK-SD-NEXT:    smull v2.8h, v2.8b, v3.8b575; CHECK-SD-NEXT:    smull v0.8h, v0.8b, v1.8b576; CHECK-SD-NEXT:    movi v1.2d, #0000000000000000577; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000578; CHECK-SD-NEXT:    sshll2 v4.4s, v0.8h, #0579; CHECK-SD-NEXT:    sshll2 v5.4s, v2.8h, #0580; CHECK-SD-NEXT:    mov v3.s[0], v4.s[0]581; CHECK-SD-NEXT:    mov v1.s[0], v5.s[0]582; CHECK-SD-NEXT:    saddw v0.4s, v3.4s, v0.4h583; CHECK-SD-NEXT:    saddw v1.4s, v1.4s, v2.4h584; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s585; CHECK-SD-NEXT:    addv s0, v0.4s586; CHECK-SD-NEXT:    fmov w0, s0587; CHECK-SD-NEXT:    ret588;589; CHECK-GI-LABEL: test_sdot_v5i8_double:590; CHECK-GI:       // %bb.0: // %entry591; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0592; CHECK-GI-NEXT:    mov b17, v0.b[1]593; CHECK-GI-NEXT:    fmov w8, s0594; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1595; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2596; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3597; CHECK-GI-NEXT:    fmov w11, s1598; CHECK-GI-NEXT:    mov b25, v1.b[1]599; CHECK-GI-NEXT:    mov b16, v1.b[2]600; CHECK-GI-NEXT:    mov b7, v1.b[3]601; CHECK-GI-NEXT:    mov b5, v1.b[4]602; CHECK-GI-NEXT:    mov b22, v2.b[1]603; CHECK-GI-NEXT:    mov b23, v3.b[1]604; CHECK-GI-NEXT:    sxtb w9, w8605; CHECK-GI-NEXT:    sxtb w11, w11606; CHECK-GI-NEXT:    mov b24, v0.b[2]607; CHECK-GI-NEXT:    fmov w8, s17608; CHECK-GI-NEXT:    mov b6, v0.b[3]609; CHECK-GI-NEXT:    mov b4, v0.b[4]610; CHECK-GI-NEXT:    fmov s1, w9611; CHECK-GI-NEXT:    mov b18, v2.b[2]612; CHECK-GI-NEXT:    mov b19, v2.b[3]613; CHECK-GI-NEXT:    mov b0, v2.b[4]614; CHECK-GI-NEXT:    fmov w9, s25615; CHECK-GI-NEXT:    fmov w12, s22616; CHECK-GI-NEXT:    sxtb w10, w8617; CHECK-GI-NEXT:    mov b21, v3.b[2]618; CHECK-GI-NEXT:    fmov w13, s23619; CHECK-GI-NEXT:    mov b20, v3.b[3]620; CHECK-GI-NEXT:    mov b17, v3.b[4]621; CHECK-GI-NEXT:    fmov w8, s24622; CHECK-GI-NEXT:    sxtb w9, w9623; CHECK-GI-NEXT:    sxtb w12, w12624; CHECK-GI-NEXT:    mov v1.h[1], w10625; CHECK-GI-NEXT:    sxtb w13, w13626; CHECK-GI-NEXT:    fmov w10, s2627; CHECK-GI-NEXT:    fmov s2, w11628; CHECK-GI-NEXT:    sxtb w8, w8629; CHECK-GI-NEXT:    fmov w11, s3630; CHECK-GI-NEXT:    sxtb w10, w10631; CHECK-GI-NEXT:    mov v2.h[1], w9632; CHECK-GI-NEXT:    fmov w9, s16633; CHECK-GI-NEXT:    sxtb w11, w11634; CHECK-GI-NEXT:    mov v1.h[2], w8635; CHECK-GI-NEXT:    fmov w8, s7636; CHECK-GI-NEXT:    fmov s3, w10637; CHECK-GI-NEXT:    fmov w10, s18638; CHECK-GI-NEXT:    sxtb w9, w9639; CHECK-GI-NEXT:    fmov s22, w11640; CHECK-GI-NEXT:    fmov w11, s21641; CHECK-GI-NEXT:    sxtb w8, w8642; CHECK-GI-NEXT:    mov v3.h[1], w12643; CHECK-GI-NEXT:    sxtb w10, w10644; CHECK-GI-NEXT:    mov v2.h[2], w9645; CHECK-GI-NEXT:    mov v22.h[1], w13646; CHECK-GI-NEXT:    sxtb w11, w11647; CHECK-GI-NEXT:    fmov w9, s19648; CHECK-GI-NEXT:    fmov w12, s6649; CHECK-GI-NEXT:    mov v3.h[2], w10650; CHECK-GI-NEXT:    fmov w10, s20651; CHECK-GI-NEXT:    sxtb w9, w9652; CHECK-GI-NEXT:    mov v22.h[2], w11653; CHECK-GI-NEXT:    sxtb w12, w12654; CHECK-GI-NEXT:    fmov w11, s4655; CHECK-GI-NEXT:    mov v2.h[3], w8656; CHECK-GI-NEXT:    fmov w8, s5657; CHECK-GI-NEXT:    sxtb w10, w10658; CHECK-GI-NEXT:    mov v1.h[3], w12659; CHECK-GI-NEXT:    mov v3.h[3], w9660; CHECK-GI-NEXT:    fmov w9, s0661; CHECK-GI-NEXT:    sxtb w11, w11662; CHECK-GI-NEXT:    mov v22.h[3], w10663; CHECK-GI-NEXT:    fmov w10, s17664; CHECK-GI-NEXT:    sxtb w8, w8665; CHECK-GI-NEXT:    sxtb w9, w9666; CHECK-GI-NEXT:    mov v1.h[4], w11667; CHECK-GI-NEXT:    mov v2.h[4], w8668; CHECK-GI-NEXT:    sxtb w10, w10669; CHECK-GI-NEXT:    mov v3.h[4], w9670; CHECK-GI-NEXT:    mov v22.h[4], w10671; CHECK-GI-NEXT:    mul v0.8h, v1.8h, v2.8h672; CHECK-GI-NEXT:    mul v1.8h, v3.8h, v22.8h673; CHECK-GI-NEXT:    smov w8, v0.h[0]674; CHECK-GI-NEXT:    smov w9, v0.h[4]675; CHECK-GI-NEXT:    smov w11, v0.h[1]676; CHECK-GI-NEXT:    smov w10, v1.h[0]677; CHECK-GI-NEXT:    smov w12, v1.h[4]678; CHECK-GI-NEXT:    smov w13, v1.h[1]679; CHECK-GI-NEXT:    fmov s2, w8680; CHECK-GI-NEXT:    fmov s3, w9681; CHECK-GI-NEXT:    smov w8, v0.h[2]682; CHECK-GI-NEXT:    smov w9, v1.h[2]683; CHECK-GI-NEXT:    fmov s4, w10684; CHECK-GI-NEXT:    fmov s5, w12685; CHECK-GI-NEXT:    mov v2.s[1], w11686; CHECK-GI-NEXT:    mov v3.s[1], wzr687; CHECK-GI-NEXT:    smov w10, v0.h[3]688; CHECK-GI-NEXT:    smov w11, v1.h[3]689; CHECK-GI-NEXT:    mov v4.s[1], w13690; CHECK-GI-NEXT:    mov v5.s[1], wzr691; CHECK-GI-NEXT:    mov v2.s[2], w8692; CHECK-GI-NEXT:    mov v3.s[2], wzr693; CHECK-GI-NEXT:    mov v4.s[2], w9694; CHECK-GI-NEXT:    mov v5.s[2], wzr695; CHECK-GI-NEXT:    mov v2.s[3], w10696; CHECK-GI-NEXT:    mov v3.s[3], wzr697; CHECK-GI-NEXT:    mov v4.s[3], w11698; CHECK-GI-NEXT:    mov v5.s[3], wzr699; CHECK-GI-NEXT:    add v0.4s, v2.4s, v3.4s700; CHECK-GI-NEXT:    add v1.4s, v4.4s, v5.4s701; CHECK-GI-NEXT:    addv s0, v0.4s702; CHECK-GI-NEXT:    addv s1, v1.4s703; CHECK-GI-NEXT:    fmov w8, s0704; CHECK-GI-NEXT:    fmov w9, s1705; CHECK-GI-NEXT:    add w0, w8, w9706; CHECK-GI-NEXT:    ret707entry:708  %az = sext <5 x i8> %a to <5 x i32>709  %bz = sext <5 x i8> %b to <5 x i32>710  %m1 = mul nuw nsw <5 x i32> %az, %bz711  %r1 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m1)712  %cz = sext <5 x i8> %c to <5 x i32>713  %dz = sext <5 x i8> %d to <5 x i32>714  %m2 = mul nuw nsw <5 x i32> %cz, %dz715  %r2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m2)716  %x = add i32 %r1, %r2717  ret i32 %x718}719 720define i32 @test_sdot_v5i8_double_nomla(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) {721; CHECK-SD-LABEL: test_sdot_v5i8_double_nomla:722; CHECK-SD:       // %bb.0: // %entry723; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #0724; CHECK-SD-NEXT:    sshll v1.8h, v2.8b, #0725; CHECK-SD-NEXT:    movi v2.2d, #0000000000000000726; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000727; CHECK-SD-NEXT:    sshll2 v4.4s, v0.8h, #0728; CHECK-SD-NEXT:    sshll2 v5.4s, v1.8h, #0729; CHECK-SD-NEXT:    mov v3.s[0], v4.s[0]730; CHECK-SD-NEXT:    mov v2.s[0], v5.s[0]731; CHECK-SD-NEXT:    saddw v0.4s, v3.4s, v0.4h732; CHECK-SD-NEXT:    saddw v1.4s, v2.4s, v1.4h733; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s734; CHECK-SD-NEXT:    addv s0, v0.4s735; CHECK-SD-NEXT:    fmov w0, s0736; CHECK-SD-NEXT:    ret737;738; CHECK-GI-LABEL: test_sdot_v5i8_double_nomla:739; CHECK-GI:       // %bb.0: // %entry740; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0741; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2742; CHECK-GI-NEXT:    smov w8, v0.b[0]743; CHECK-GI-NEXT:    smov w9, v0.b[4]744; CHECK-GI-NEXT:    smov w10, v2.b[0]745; CHECK-GI-NEXT:    smov w12, v2.b[4]746; CHECK-GI-NEXT:    smov w11, v0.b[1]747; CHECK-GI-NEXT:    smov w13, v2.b[1]748; CHECK-GI-NEXT:    fmov s1, w8749; CHECK-GI-NEXT:    fmov s3, w9750; CHECK-GI-NEXT:    fmov s4, w10751; CHECK-GI-NEXT:    fmov s5, w12752; CHECK-GI-NEXT:    smov w8, v0.b[2]753; CHECK-GI-NEXT:    smov w9, v2.b[2]754; CHECK-GI-NEXT:    smov w10, v0.b[3]755; CHECK-GI-NEXT:    mov v1.s[1], w11756; CHECK-GI-NEXT:    mov v3.s[1], wzr757; CHECK-GI-NEXT:    mov v4.s[1], w13758; CHECK-GI-NEXT:    mov v5.s[1], wzr759; CHECK-GI-NEXT:    smov w11, v2.b[3]760; CHECK-GI-NEXT:    mov v1.s[2], w8761; CHECK-GI-NEXT:    mov v3.s[2], wzr762; CHECK-GI-NEXT:    mov v4.s[2], w9763; CHECK-GI-NEXT:    mov v5.s[2], wzr764; CHECK-GI-NEXT:    mov v1.s[3], w10765; CHECK-GI-NEXT:    mov v3.s[3], wzr766; CHECK-GI-NEXT:    mov v4.s[3], w11767; CHECK-GI-NEXT:    mov v5.s[3], wzr768; CHECK-GI-NEXT:    add v0.4s, v1.4s, v3.4s769; CHECK-GI-NEXT:    add v1.4s, v4.4s, v5.4s770; CHECK-GI-NEXT:    addv s0, v0.4s771; CHECK-GI-NEXT:    addv s1, v1.4s772; CHECK-GI-NEXT:    fmov w8, s0773; CHECK-GI-NEXT:    fmov w9, s1774; CHECK-GI-NEXT:    add w0, w8, w9775; CHECK-GI-NEXT:    ret776entry:777  %az = sext <5 x i8> %a to <5 x i32>778  %r1 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %az)779  %cz = sext <5 x i8> %c to <5 x i32>780  %r2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %cz)781  %x = add i32 %r1, %r2782  ret i32 %x783}784 785define i32 @test_udot_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) {786; CHECK-LABEL: test_udot_v8i8:787; CHECK:       // %bb.0: // %entry788; CHECK-NEXT:    movi v0.2d, #0000000000000000789; CHECK-NEXT:    ldr d1, [x0]790; CHECK-NEXT:    ldr d2, [x1]791; CHECK-NEXT:    udot v0.2s, v2.8b, v1.8b792; CHECK-NEXT:    addp v0.2s, v0.2s, v0.2s793; CHECK-NEXT:    fmov w0, s0794; CHECK-NEXT:    ret795entry:796  %0 = load <8 x i8>, ptr %a797  %1 = zext <8 x i8> %0 to <8 x i32>798  %2 = load <8 x i8>, ptr %b799  %3 = zext <8 x i8> %2 to <8 x i32>800  %4 = mul nuw nsw <8 x i32> %3, %1801  %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)802  ret i32 %5803}804 805define i32 @test_udot_v8i8_nomla(ptr nocapture readonly %a1) {806; CHECK-SD-LABEL: test_udot_v8i8_nomla:807; CHECK-SD:       // %bb.0: // %entry808; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000809; CHECK-SD-NEXT:    movi v1.8b, #1810; CHECK-SD-NEXT:    ldr d2, [x0]811; CHECK-SD-NEXT:    udot v0.2s, v2.8b, v1.8b812; CHECK-SD-NEXT:    addp v0.2s, v0.2s, v0.2s813; CHECK-SD-NEXT:    fmov w0, s0814; CHECK-SD-NEXT:    ret815;816; CHECK-GI-LABEL: test_udot_v8i8_nomla:817; CHECK-GI:       // %bb.0: // %entry818; CHECK-GI-NEXT:    movi v0.8b, #1819; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000820; CHECK-GI-NEXT:    ldr d2, [x0]821; CHECK-GI-NEXT:    udot v1.2s, v2.8b, v0.8b822; CHECK-GI-NEXT:    addp v0.2s, v1.2s, v1.2s823; CHECK-GI-NEXT:    fmov w0, s0824; CHECK-GI-NEXT:    ret825entry:826  %0 = load <8 x i8>, ptr %a1827  %1 = zext <8 x i8> %0 to <8 x i32>828  %2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)829  ret i32 %2830}831 832define i32 @test_sdot_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) {833; CHECK-LABEL: test_sdot_v8i8:834; CHECK:       // %bb.0: // %entry835; CHECK-NEXT:    movi v0.2d, #0000000000000000836; CHECK-NEXT:    ldr d1, [x0]837; CHECK-NEXT:    ldr d2, [x1]838; CHECK-NEXT:    sdot v0.2s, v2.8b, v1.8b839; CHECK-NEXT:    addp v0.2s, v0.2s, v0.2s840; CHECK-NEXT:    fmov w0, s0841; CHECK-NEXT:    ret842entry:843  %0 = load <8 x i8>, ptr %a844  %1 = sext <8 x i8> %0 to <8 x i32>845  %2 = load <8 x i8>, ptr %b846  %3 = sext <8 x i8> %2 to <8 x i32>847  %4 = mul nsw <8 x i32> %3, %1848  %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)849  ret i32 %5850}851 852define i32 @test_sdot_v8i8_nomla(ptr nocapture readonly %a1) {853; CHECK-SD-LABEL: test_sdot_v8i8_nomla:854; CHECK-SD:       // %bb.0: // %entry855; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000856; CHECK-SD-NEXT:    movi v1.8b, #1857; CHECK-SD-NEXT:    ldr d2, [x0]858; CHECK-SD-NEXT:    sdot v0.2s, v2.8b, v1.8b859; CHECK-SD-NEXT:    addp v0.2s, v0.2s, v0.2s860; CHECK-SD-NEXT:    fmov w0, s0861; CHECK-SD-NEXT:    ret862;863; CHECK-GI-LABEL: test_sdot_v8i8_nomla:864; CHECK-GI:       // %bb.0: // %entry865; CHECK-GI-NEXT:    movi v0.8b, #1866; CHECK-GI-NEXT:    movi v1.2d, #0000000000000000867; CHECK-GI-NEXT:    ldr d2, [x0]868; CHECK-GI-NEXT:    sdot v1.2s, v2.8b, v0.8b869; CHECK-GI-NEXT:    addp v0.2s, v1.2s, v1.2s870; CHECK-GI-NEXT:    fmov w0, s0871; CHECK-GI-NEXT:    ret872entry:873  %0 = load <8 x i8>, ptr %a1874  %1 = sext <8 x i8> %0 to <8 x i32>875  %2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)876  ret i32 %2877}878 879define i32 @test_usdot_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) {880; CHECK-SD-LABEL: test_usdot_v8i8:881; CHECK-SD:       // %bb.0: // %entry882; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000883; CHECK-SD-NEXT:    ldr d1, [x0]884; CHECK-SD-NEXT:    ldr d2, [x1]885; CHECK-SD-NEXT:    usdot v0.2s, v1.8b, v2.8b886; CHECK-SD-NEXT:    addp v0.2s, v0.2s, v0.2s887; CHECK-SD-NEXT:    fmov w0, s0888; CHECK-SD-NEXT:    ret889;890; CHECK-GI-LABEL: test_usdot_v8i8:891; CHECK-GI:       // %bb.0: // %entry892; CHECK-GI-NEXT:    ldr d0, [x0]893; CHECK-GI-NEXT:    ldr d1, [x1]894; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0895; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #0896; CHECK-GI-NEXT:    smull v2.4s, v1.4h, v0.4h897; CHECK-GI-NEXT:    smlal2 v2.4s, v1.8h, v0.8h898; CHECK-GI-NEXT:    addv s0, v2.4s899; CHECK-GI-NEXT:    fmov w0, s0900; CHECK-GI-NEXT:    ret901entry:902  %0 = load <8 x i8>, ptr %a903  %1 = zext <8 x i8> %0 to <8 x i32>904  %2 = load <8 x i8>, ptr %b905  %3 = sext <8 x i8> %2 to <8 x i32>906  %4 = mul nsw <8 x i32> %3, %1907  %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)908  ret i32 %5909}910 911define i32 @test_usdot_swapped_operands_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) {912; CHECK-SD-LABEL: test_usdot_swapped_operands_v8i8:913; CHECK-SD:       // %bb.0: // %entry914; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000915; CHECK-SD-NEXT:    ldr d1, [x0]916; CHECK-SD-NEXT:    ldr d2, [x1]917; CHECK-SD-NEXT:    usdot v0.2s, v2.8b, v1.8b918; CHECK-SD-NEXT:    addp v0.2s, v0.2s, v0.2s919; CHECK-SD-NEXT:    fmov w0, s0920; CHECK-SD-NEXT:    ret921;922; CHECK-GI-LABEL: test_usdot_swapped_operands_v8i8:923; CHECK-GI:       // %bb.0: // %entry924; CHECK-GI-NEXT:    ldr d0, [x0]925; CHECK-GI-NEXT:    ldr d1, [x1]926; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0927; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0928; CHECK-GI-NEXT:    smull v2.4s, v1.4h, v0.4h929; CHECK-GI-NEXT:    smlal2 v2.4s, v1.8h, v0.8h930; CHECK-GI-NEXT:    addv s0, v2.4s931; CHECK-GI-NEXT:    fmov w0, s0932; CHECK-GI-NEXT:    ret933entry:934  %0 = load <8 x i8>, ptr %a935  %1 = sext <8 x i8> %0 to <8 x i32>936  %2 = load <8 x i8>, ptr %b937  %3 = zext <8 x i8> %2 to <8 x i32>938  %4 = mul nsw <8 x i32> %3, %1939  %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)940  ret i32 %5941}942 943define i32 @test_udot_v16i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {944; CHECK-LABEL: test_udot_v16i8:945; CHECK:       // %bb.0: // %entry946; CHECK-NEXT:    movi v0.2d, #0000000000000000947; CHECK-NEXT:    ldr q1, [x0]948; CHECK-NEXT:    ldr q2, [x1]949; CHECK-NEXT:    udot v0.4s, v2.16b, v1.16b950; CHECK-NEXT:    addv s0, v0.4s951; CHECK-NEXT:    fmov w8, s0952; CHECK-NEXT:    add w0, w8, w2953; CHECK-NEXT:    ret954entry:955  %0 = load <16 x i8>, ptr %a956  %1 = zext <16 x i8> %0 to <16 x i32>957  %2 = load <16 x i8>, ptr %b958  %3 = zext <16 x i8> %2 to <16 x i32>959  %4 = mul nuw nsw <16 x i32> %3, %1960  %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)961  %op.extra = add i32 %5, %sum962  ret i32 %op.extra963}964 965define i32 @test_udot_v16i8_nomla(ptr nocapture readonly %a1) {966; CHECK-LABEL: test_udot_v16i8_nomla:967; CHECK:       // %bb.0: // %entry968; CHECK-NEXT:    movi v0.16b, #1969; CHECK-NEXT:    movi v1.2d, #0000000000000000970; CHECK-NEXT:    ldr q2, [x0]971; CHECK-NEXT:    udot v1.4s, v2.16b, v0.16b972; CHECK-NEXT:    addv s0, v1.4s973; CHECK-NEXT:    fmov w0, s0974; CHECK-NEXT:    ret975entry:976  %0 = load <16 x i8>, ptr %a1977  %1 = zext <16 x i8> %0 to <16 x i32>978  %2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)979  ret i32 %2980}981 982define i32 @test_sdot_v16i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {983; CHECK-LABEL: test_sdot_v16i8:984; CHECK:       // %bb.0: // %entry985; CHECK-NEXT:    movi v0.2d, #0000000000000000986; CHECK-NEXT:    ldr q1, [x0]987; CHECK-NEXT:    ldr q2, [x1]988; CHECK-NEXT:    sdot v0.4s, v2.16b, v1.16b989; CHECK-NEXT:    addv s0, v0.4s990; CHECK-NEXT:    fmov w8, s0991; CHECK-NEXT:    add w0, w8, w2992; CHECK-NEXT:    ret993entry:994  %0 = load <16 x i8>, ptr %a995  %1 = sext <16 x i8> %0 to <16 x i32>996  %2 = load <16 x i8>, ptr %b997  %3 = sext <16 x i8> %2 to <16 x i32>998  %4 = mul nsw <16 x i32> %3, %1999  %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)1000  %op.extra = add nsw i32 %5, %sum1001  ret i32 %op.extra1002}1003 1004define i32 @test_sdot_v16i8_nomla(ptr nocapture readonly %a1) {1005; CHECK-LABEL: test_sdot_v16i8_nomla:1006; CHECK:       // %bb.0: // %entry1007; CHECK-NEXT:    movi v0.16b, #11008; CHECK-NEXT:    movi v1.2d, #00000000000000001009; CHECK-NEXT:    ldr q2, [x0]1010; CHECK-NEXT:    sdot v1.4s, v2.16b, v0.16b1011; CHECK-NEXT:    addv s0, v1.4s1012; CHECK-NEXT:    fmov w0, s01013; CHECK-NEXT:    ret1014entry:1015  %0 = load <16 x i8>, ptr %a11016  %1 = sext <16 x i8> %0 to <16 x i32>1017  %2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)1018  ret i32 %21019}1020 1021define i32 @test_usdot_v16i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {1022; CHECK-SD-LABEL: test_usdot_v16i8:1023; CHECK-SD:       // %bb.0: // %entry1024; CHECK-SD-NEXT:    movi v0.2d, #00000000000000001025; CHECK-SD-NEXT:    ldr q1, [x0]1026; CHECK-SD-NEXT:    ldr q2, [x1]1027; CHECK-SD-NEXT:    usdot v0.4s, v1.16b, v2.16b1028; CHECK-SD-NEXT:    addv s0, v0.4s1029; CHECK-SD-NEXT:    fmov w8, s01030; CHECK-SD-NEXT:    add w0, w8, w21031; CHECK-SD-NEXT:    ret1032;1033; CHECK-GI-LABEL: test_usdot_v16i8:1034; CHECK-GI:       // %bb.0: // %entry1035; CHECK-GI-NEXT:    ldr q0, [x0]1036; CHECK-GI-NEXT:    ldr q1, [x1]1037; CHECK-GI-NEXT:    ushll v2.8h, v0.8b, #01038; CHECK-GI-NEXT:    ushll2 v0.8h, v0.16b, #01039; CHECK-GI-NEXT:    sshll v3.8h, v1.8b, #01040; CHECK-GI-NEXT:    sshll2 v1.8h, v1.16b, #01041; CHECK-GI-NEXT:    smull v4.4s, v3.4h, v2.4h1042; CHECK-GI-NEXT:    smull v5.4s, v1.4h, v0.4h1043; CHECK-GI-NEXT:    smlal2 v4.4s, v3.8h, v2.8h1044; CHECK-GI-NEXT:    smlal2 v5.4s, v1.8h, v0.8h1045; CHECK-GI-NEXT:    add v0.4s, v4.4s, v5.4s1046; CHECK-GI-NEXT:    addv s0, v0.4s1047; CHECK-GI-NEXT:    fmov w8, s01048; CHECK-GI-NEXT:    add w0, w8, w21049; CHECK-GI-NEXT:    ret1050entry:1051  %0 = load <16 x i8>, ptr %a1052  %1 = zext <16 x i8> %0 to <16 x i32>1053  %2 = load <16 x i8>, ptr %b1054  %3 = sext <16 x i8> %2 to <16 x i32>1055  %4 = mul nsw <16 x i32> %3, %11056  %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)1057  %op.extra = add nsw i32 %5, %sum1058  ret i32 %op.extra1059}1060 1061define i32 @test_usdot_swapped_operands_v16i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {1062; CHECK-SD-LABEL: test_usdot_swapped_operands_v16i8:1063; CHECK-SD:       // %bb.0: // %entry1064; CHECK-SD-NEXT:    movi v0.2d, #00000000000000001065; CHECK-SD-NEXT:    ldr q1, [x0]1066; CHECK-SD-NEXT:    ldr q2, [x1]1067; CHECK-SD-NEXT:    usdot v0.4s, v2.16b, v1.16b1068; CHECK-SD-NEXT:    addv s0, v0.4s1069; CHECK-SD-NEXT:    fmov w8, s01070; CHECK-SD-NEXT:    add w0, w8, w21071; CHECK-SD-NEXT:    ret1072;1073; CHECK-GI-LABEL: test_usdot_swapped_operands_v16i8:1074; CHECK-GI:       // %bb.0: // %entry1075; CHECK-GI-NEXT:    ldr q0, [x0]1076; CHECK-GI-NEXT:    ldr q1, [x1]1077; CHECK-GI-NEXT:    sshll v2.8h, v0.8b, #01078; CHECK-GI-NEXT:    sshll2 v0.8h, v0.16b, #01079; CHECK-GI-NEXT:    ushll v3.8h, v1.8b, #01080; CHECK-GI-NEXT:    ushll2 v1.8h, v1.16b, #01081; CHECK-GI-NEXT:    smull v4.4s, v3.4h, v2.4h1082; CHECK-GI-NEXT:    smull v5.4s, v1.4h, v0.4h1083; CHECK-GI-NEXT:    smlal2 v4.4s, v3.8h, v2.8h1084; CHECK-GI-NEXT:    smlal2 v5.4s, v1.8h, v0.8h1085; CHECK-GI-NEXT:    add v0.4s, v4.4s, v5.4s1086; CHECK-GI-NEXT:    addv s0, v0.4s1087; CHECK-GI-NEXT:    fmov w8, s01088; CHECK-GI-NEXT:    add w0, w8, w21089; CHECK-GI-NEXT:    ret1090entry:1091  %0 = load <16 x i8>, ptr %a1092  %1 = sext <16 x i8> %0 to <16 x i32>1093  %2 = load <16 x i8>, ptr %b1094  %3 = zext <16 x i8> %2 to <16 x i32>1095  %4 = mul nsw <16 x i32> %3, %11096  %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)1097  %op.extra = add nsw i32 %5, %sum1098  ret i32 %op.extra1099}1100 1101define i32 @test_udot_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {1102; CHECK-SD-LABEL: test_udot_v8i8_double:1103; CHECK-SD:       // %bb.0: // %entry1104; CHECK-SD-NEXT:    movi v4.2d, #00000000000000001105; CHECK-SD-NEXT:    udot v4.2s, v2.8b, v3.8b1106; CHECK-SD-NEXT:    udot v4.2s, v0.8b, v1.8b1107; CHECK-SD-NEXT:    addp v0.2s, v4.2s, v4.2s1108; CHECK-SD-NEXT:    fmov w0, s01109; CHECK-SD-NEXT:    ret1110;1111; CHECK-GI-LABEL: test_udot_v8i8_double:1112; CHECK-GI:       // %bb.0: // %entry1113; CHECK-GI-NEXT:    movi v4.2d, #00000000000000001114; CHECK-GI-NEXT:    movi v5.2d, #00000000000000001115; CHECK-GI-NEXT:    udot v5.2s, v0.8b, v1.8b1116; CHECK-GI-NEXT:    udot v4.2s, v2.8b, v3.8b1117; CHECK-GI-NEXT:    addp v0.2s, v5.2s, v5.2s1118; CHECK-GI-NEXT:    addp v1.2s, v4.2s, v4.2s1119; CHECK-GI-NEXT:    fmov w8, s01120; CHECK-GI-NEXT:    fmov w9, s11121; CHECK-GI-NEXT:    add w0, w8, w91122; CHECK-GI-NEXT:    ret1123entry:1124  %az = zext <8 x i8> %a to <8 x i32>1125  %bz = zext <8 x i8> %b to <8 x i32>1126  %m1 = mul nuw nsw <8 x i32> %az, %bz1127  %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m1)1128  %cz = zext <8 x i8> %c to <8 x i32>1129  %dz = zext <8 x i8> %d to <8 x i32>1130  %m2 = mul nuw nsw <8 x i32> %cz, %dz1131  %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m2)1132  %x = add i32 %r1, %r21133  ret i32 %x1134}1135 1136define i32 @test_udot_v8i8_double_nomla(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {1137; CHECK-SD-LABEL: test_udot_v8i8_double_nomla:1138; CHECK-SD:       // %bb.0: // %entry1139; CHECK-SD-NEXT:    movi v1.2d, #00000000000000001140; CHECK-SD-NEXT:    movi v3.8b, #11141; CHECK-SD-NEXT:    udot v1.2s, v2.8b, v3.8b1142; CHECK-SD-NEXT:    udot v1.2s, v0.8b, v3.8b1143; CHECK-SD-NEXT:    addp v0.2s, v1.2s, v1.2s1144; CHECK-SD-NEXT:    fmov w0, s01145; CHECK-SD-NEXT:    ret1146;1147; CHECK-GI-LABEL: test_udot_v8i8_double_nomla:1148; CHECK-GI:       // %bb.0: // %entry1149; CHECK-GI-NEXT:    movi v1.8b, #11150; CHECK-GI-NEXT:    movi v3.2d, #00000000000000001151; CHECK-GI-NEXT:    movi v4.2d, #00000000000000001152; CHECK-GI-NEXT:    udot v4.2s, v0.8b, v1.8b1153; CHECK-GI-NEXT:    udot v3.2s, v2.8b, v1.8b1154; CHECK-GI-NEXT:    addp v0.2s, v4.2s, v4.2s1155; CHECK-GI-NEXT:    addp v1.2s, v3.2s, v3.2s1156; CHECK-GI-NEXT:    fmov w8, s01157; CHECK-GI-NEXT:    fmov w9, s11158; CHECK-GI-NEXT:    add w0, w8, w91159; CHECK-GI-NEXT:    ret1160entry:1161  %az = zext <8 x i8> %a to <8 x i32>1162  %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %az)1163  %cz = zext <8 x i8> %c to <8 x i32>1164  %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %cz)1165  %x = add i32 %r1, %r21166  ret i32 %x1167}1168 1169define i32 @test_udot_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {1170; CHECK-SD-LABEL: test_udot_v16i8_double:1171; CHECK-SD:       // %bb.0: // %entry1172; CHECK-SD-NEXT:    movi v4.2d, #00000000000000001173; CHECK-SD-NEXT:    udot v4.4s, v2.16b, v3.16b1174; CHECK-SD-NEXT:    udot v4.4s, v0.16b, v1.16b1175; CHECK-SD-NEXT:    addv s0, v4.4s1176; CHECK-SD-NEXT:    fmov w0, s01177; CHECK-SD-NEXT:    ret1178;1179; CHECK-GI-LABEL: test_udot_v16i8_double:1180; CHECK-GI:       // %bb.0: // %entry1181; CHECK-GI-NEXT:    movi v4.2d, #00000000000000001182; CHECK-GI-NEXT:    movi v5.2d, #00000000000000001183; CHECK-GI-NEXT:    udot v5.4s, v0.16b, v1.16b1184; CHECK-GI-NEXT:    udot v4.4s, v2.16b, v3.16b1185; CHECK-GI-NEXT:    addv s0, v5.4s1186; CHECK-GI-NEXT:    addv s1, v4.4s1187; CHECK-GI-NEXT:    fmov w8, s01188; CHECK-GI-NEXT:    fmov w9, s11189; CHECK-GI-NEXT:    add w0, w8, w91190; CHECK-GI-NEXT:    ret1191entry:1192  %az = zext <16 x i8> %a to <16 x i32>1193  %bz = zext <16 x i8> %b to <16 x i32>1194  %m1 = mul nuw nsw <16 x i32> %az, %bz1195  %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m1)1196  %cz = zext <16 x i8> %c to <16 x i32>1197  %dz = zext <16 x i8> %d to <16 x i32>1198  %m2 = mul nuw nsw <16 x i32> %cz, %dz1199  %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m2)1200  %x = add i32 %r1, %r21201  ret i32 %x1202}1203 1204define i32 @test_udot_v16i8_double_nomla(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {1205; CHECK-SD-LABEL: test_udot_v16i8_double_nomla:1206; CHECK-SD:       // %bb.0: // %entry1207; CHECK-SD-NEXT:    movi v1.16b, #11208; CHECK-SD-NEXT:    movi v3.2d, #00000000000000001209; CHECK-SD-NEXT:    udot v3.4s, v2.16b, v1.16b1210; CHECK-SD-NEXT:    udot v3.4s, v0.16b, v1.16b1211; CHECK-SD-NEXT:    addv s0, v3.4s1212; CHECK-SD-NEXT:    fmov w0, s01213; CHECK-SD-NEXT:    ret1214;1215; CHECK-GI-LABEL: test_udot_v16i8_double_nomla:1216; CHECK-GI:       // %bb.0: // %entry1217; CHECK-GI-NEXT:    movi v1.16b, #11218; CHECK-GI-NEXT:    movi v3.2d, #00000000000000001219; CHECK-GI-NEXT:    movi v4.2d, #00000000000000001220; CHECK-GI-NEXT:    udot v4.4s, v0.16b, v1.16b1221; CHECK-GI-NEXT:    udot v3.4s, v2.16b, v1.16b1222; CHECK-GI-NEXT:    addv s0, v4.4s1223; CHECK-GI-NEXT:    addv s1, v3.4s1224; CHECK-GI-NEXT:    fmov w8, s01225; CHECK-GI-NEXT:    fmov w9, s11226; CHECK-GI-NEXT:    add w0, w8, w91227; CHECK-GI-NEXT:    ret1228entry:1229  %az = zext <16 x i8> %a to <16 x i32>1230  %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %az)1231  %cz = zext <16 x i8> %c to <16 x i32>1232  %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %cz)1233  %x = add i32 %r1, %r21234  ret i32 %x1235}1236 1237define i32 @test_sdot_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {1238; CHECK-SD-LABEL: test_sdot_v8i8_double:1239; CHECK-SD:       // %bb.0: // %entry1240; CHECK-SD-NEXT:    movi v4.2d, #00000000000000001241; CHECK-SD-NEXT:    sdot v4.2s, v2.8b, v3.8b1242; CHECK-SD-NEXT:    sdot v4.2s, v0.8b, v1.8b1243; CHECK-SD-NEXT:    addp v0.2s, v4.2s, v4.2s1244; CHECK-SD-NEXT:    fmov w0, s01245; CHECK-SD-NEXT:    ret1246;1247; CHECK-GI-LABEL: test_sdot_v8i8_double:1248; CHECK-GI:       // %bb.0: // %entry1249; CHECK-GI-NEXT:    movi v4.2d, #00000000000000001250; CHECK-GI-NEXT:    movi v5.2d, #00000000000000001251; CHECK-GI-NEXT:    sdot v5.2s, v0.8b, v1.8b1252; CHECK-GI-NEXT:    sdot v4.2s, v2.8b, v3.8b1253; CHECK-GI-NEXT:    addp v0.2s, v5.2s, v5.2s1254; CHECK-GI-NEXT:    addp v1.2s, v4.2s, v4.2s1255; CHECK-GI-NEXT:    fmov w8, s01256; CHECK-GI-NEXT:    fmov w9, s11257; CHECK-GI-NEXT:    add w0, w8, w91258; CHECK-GI-NEXT:    ret1259entry:1260  %az = sext <8 x i8> %a to <8 x i32>1261  %bz = sext <8 x i8> %b to <8 x i32>1262  %m1 = mul nuw nsw <8 x i32> %az, %bz1263  %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m1)1264  %cz = sext <8 x i8> %c to <8 x i32>1265  %dz = sext <8 x i8> %d to <8 x i32>1266  %m2 = mul nuw nsw <8 x i32> %cz, %dz1267  %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m2)1268  %x = add i32 %r1, %r21269  ret i32 %x1270}1271 1272define i32 @test_sdot_v8i8_double_nomla(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {1273; CHECK-SD-LABEL: test_sdot_v8i8_double_nomla:1274; CHECK-SD:       // %bb.0: // %entry1275; CHECK-SD-NEXT:    movi v1.2d, #00000000000000001276; CHECK-SD-NEXT:    movi v3.8b, #11277; CHECK-SD-NEXT:    sdot v1.2s, v2.8b, v3.8b1278; CHECK-SD-NEXT:    sdot v1.2s, v0.8b, v3.8b1279; CHECK-SD-NEXT:    addp v0.2s, v1.2s, v1.2s1280; CHECK-SD-NEXT:    fmov w0, s01281; CHECK-SD-NEXT:    ret1282;1283; CHECK-GI-LABEL: test_sdot_v8i8_double_nomla:1284; CHECK-GI:       // %bb.0: // %entry1285; CHECK-GI-NEXT:    movi v1.8b, #11286; CHECK-GI-NEXT:    movi v3.2d, #00000000000000001287; CHECK-GI-NEXT:    movi v4.2d, #00000000000000001288; CHECK-GI-NEXT:    sdot v4.2s, v0.8b, v1.8b1289; CHECK-GI-NEXT:    sdot v3.2s, v2.8b, v1.8b1290; CHECK-GI-NEXT:    addp v0.2s, v4.2s, v4.2s1291; CHECK-GI-NEXT:    addp v1.2s, v3.2s, v3.2s1292; CHECK-GI-NEXT:    fmov w8, s01293; CHECK-GI-NEXT:    fmov w9, s11294; CHECK-GI-NEXT:    add w0, w8, w91295; CHECK-GI-NEXT:    ret1296entry:1297  %az = sext <8 x i8> %a to <8 x i32>1298  %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %az)1299  %cz = sext <8 x i8> %c to <8 x i32>1300  %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %cz)1301  %x = add i32 %r1, %r21302  ret i32 %x1303}1304 1305define i32 @test_sdot_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {1306; CHECK-SD-LABEL: test_sdot_v16i8_double:1307; CHECK-SD:       // %bb.0: // %entry1308; CHECK-SD-NEXT:    movi v4.2d, #00000000000000001309; CHECK-SD-NEXT:    sdot v4.4s, v2.16b, v3.16b1310; CHECK-SD-NEXT:    sdot v4.4s, v0.16b, v1.16b1311; CHECK-SD-NEXT:    addv s0, v4.4s1312; CHECK-SD-NEXT:    fmov w0, s01313; CHECK-SD-NEXT:    ret1314;1315; CHECK-GI-LABEL: test_sdot_v16i8_double:1316; CHECK-GI:       // %bb.0: // %entry1317; CHECK-GI-NEXT:    movi v4.2d, #00000000000000001318; CHECK-GI-NEXT:    movi v5.2d, #00000000000000001319; CHECK-GI-NEXT:    sdot v5.4s, v0.16b, v1.16b1320; CHECK-GI-NEXT:    sdot v4.4s, v2.16b, v3.16b1321; CHECK-GI-NEXT:    addv s0, v5.4s1322; CHECK-GI-NEXT:    addv s1, v4.4s1323; CHECK-GI-NEXT:    fmov w8, s01324; CHECK-GI-NEXT:    fmov w9, s11325; CHECK-GI-NEXT:    add w0, w8, w91326; CHECK-GI-NEXT:    ret1327entry:1328  %az = sext <16 x i8> %a to <16 x i32>1329  %bz = sext <16 x i8> %b to <16 x i32>1330  %m1 = mul nuw nsw <16 x i32> %az, %bz1331  %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m1)1332  %cz = sext <16 x i8> %c to <16 x i32>1333  %dz = sext <16 x i8> %d to <16 x i32>1334  %m2 = mul nuw nsw <16 x i32> %cz, %dz1335  %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m2)1336  %x = add i32 %r1, %r21337  ret i32 %x1338}1339 1340define i32 @test_sdot_v16i8_double_nomla(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {1341; CHECK-SD-LABEL: test_sdot_v16i8_double_nomla:1342; CHECK-SD:       // %bb.0: // %entry1343; CHECK-SD-NEXT:    movi v1.16b, #11344; CHECK-SD-NEXT:    movi v3.2d, #00000000000000001345; CHECK-SD-NEXT:    sdot v3.4s, v2.16b, v1.16b1346; CHECK-SD-NEXT:    sdot v3.4s, v0.16b, v1.16b1347; CHECK-SD-NEXT:    addv s0, v3.4s1348; CHECK-SD-NEXT:    fmov w0, s01349; CHECK-SD-NEXT:    ret1350;1351; CHECK-GI-LABEL: test_sdot_v16i8_double_nomla:1352; CHECK-GI:       // %bb.0: // %entry1353; CHECK-GI-NEXT:    movi v1.16b, #11354; CHECK-GI-NEXT:    movi v3.2d, #00000000000000001355; CHECK-GI-NEXT:    movi v4.2d, #00000000000000001356; CHECK-GI-NEXT:    sdot v4.4s, v0.16b, v1.16b1357; CHECK-GI-NEXT:    sdot v3.4s, v2.16b, v1.16b1358; CHECK-GI-NEXT:    addv s0, v4.4s1359; CHECK-GI-NEXT:    addv s1, v3.4s1360; CHECK-GI-NEXT:    fmov w8, s01361; CHECK-GI-NEXT:    fmov w9, s11362; CHECK-GI-NEXT:    add w0, w8, w91363; CHECK-GI-NEXT:    ret1364entry:1365  %az = sext <16 x i8> %a to <16 x i32>1366  %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %az)1367  %cz = sext <16 x i8> %c to <16 x i32>1368  %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %cz)1369  %x = add i32 %r1, %r21370  ret i32 %x1371}1372 1373 1374define i32 @test_usdot_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {1375; CHECK-SD-LABEL: test_usdot_v8i8_double:1376; CHECK-SD:       // %bb.0: // %entry1377; CHECK-SD-NEXT:    movi v4.2d, #00000000000000001378; CHECK-SD-NEXT:    movi v5.2d, #00000000000000001379; CHECK-SD-NEXT:    usdot v5.2s, v0.8b, v1.8b1380; CHECK-SD-NEXT:    usdot v4.2s, v2.8b, v3.8b1381; CHECK-SD-NEXT:    add v0.2s, v5.2s, v4.2s1382; CHECK-SD-NEXT:    addp v0.2s, v0.2s, v0.2s1383; CHECK-SD-NEXT:    fmov w0, s01384; CHECK-SD-NEXT:    ret1385;1386; CHECK-GI-LABEL: test_usdot_v8i8_double:1387; CHECK-GI:       // %bb.0: // %entry1388; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #01389; CHECK-GI-NEXT:    sshll v1.8h, v1.8b, #01390; CHECK-GI-NEXT:    ushll v2.8h, v2.8b, #01391; CHECK-GI-NEXT:    sshll v3.8h, v3.8b, #01392; CHECK-GI-NEXT:    smull v4.4s, v0.4h, v1.4h1393; CHECK-GI-NEXT:    smull v5.4s, v2.4h, v3.4h1394; CHECK-GI-NEXT:    smlal2 v4.4s, v0.8h, v1.8h1395; CHECK-GI-NEXT:    smlal2 v5.4s, v2.8h, v3.8h1396; CHECK-GI-NEXT:    addv s0, v4.4s1397; CHECK-GI-NEXT:    addv s1, v5.4s1398; CHECK-GI-NEXT:    fmov w8, s01399; CHECK-GI-NEXT:    fmov w9, s11400; CHECK-GI-NEXT:    add w0, w8, w91401; CHECK-GI-NEXT:    ret1402entry:1403  %az = zext <8 x i8> %a to <8 x i32>1404  %bz = sext <8 x i8> %b to <8 x i32>1405  %m1 = mul nuw nsw <8 x i32> %az, %bz1406  %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m1)1407  %cz = zext <8 x i8> %c to <8 x i32>1408  %dz = sext <8 x i8> %d to <8 x i32>1409  %m2 = mul nuw nsw <8 x i32> %cz, %dz1410  %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m2)1411  %x = add i32 %r1, %r21412  ret i32 %x1413}1414 1415define i32 @test_usdot_swapped_operands_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {1416; CHECK-SD-LABEL: test_usdot_swapped_operands_v8i8_double:1417; CHECK-SD:       // %bb.0: // %entry1418; CHECK-SD-NEXT:    movi v4.2d, #00000000000000001419; CHECK-SD-NEXT:    movi v5.2d, #00000000000000001420; CHECK-SD-NEXT:    usdot v5.2s, v1.8b, v0.8b1421; CHECK-SD-NEXT:    usdot v4.2s, v3.8b, v2.8b1422; CHECK-SD-NEXT:    add v0.2s, v5.2s, v4.2s1423; CHECK-SD-NEXT:    addp v0.2s, v0.2s, v0.2s1424; CHECK-SD-NEXT:    fmov w0, s01425; CHECK-SD-NEXT:    ret1426;1427; CHECK-GI-LABEL: test_usdot_swapped_operands_v8i8_double:1428; CHECK-GI:       // %bb.0: // %entry1429; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #01430; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #01431; CHECK-GI-NEXT:    sshll v2.8h, v2.8b, #01432; CHECK-GI-NEXT:    ushll v3.8h, v3.8b, #01433; CHECK-GI-NEXT:    smull v4.4s, v0.4h, v1.4h1434; CHECK-GI-NEXT:    smull v5.4s, v2.4h, v3.4h1435; CHECK-GI-NEXT:    smlal2 v4.4s, v0.8h, v1.8h1436; CHECK-GI-NEXT:    smlal2 v5.4s, v2.8h, v3.8h1437; CHECK-GI-NEXT:    addv s0, v4.4s1438; CHECK-GI-NEXT:    addv s1, v5.4s1439; CHECK-GI-NEXT:    fmov w8, s01440; CHECK-GI-NEXT:    fmov w9, s11441; CHECK-GI-NEXT:    add w0, w8, w91442; CHECK-GI-NEXT:    ret1443entry:1444  %az = sext <8 x i8> %a to <8 x i32>1445  %bz = zext <8 x i8> %b to <8 x i32>1446  %m1 = mul nuw nsw <8 x i32> %az, %bz1447  %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m1)1448  %cz = sext <8 x i8> %c to <8 x i32>1449  %dz = zext <8 x i8> %d to <8 x i32>1450  %m2 = mul nuw nsw <8 x i32> %cz, %dz1451  %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m2)1452  %x = add i32 %r1, %r21453  ret i32 %x1454}1455 1456define i32 @test_usdot_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {1457; CHECK-SD-LABEL: test_usdot_v16i8_double:1458; CHECK-SD:       // %bb.0: // %entry1459; CHECK-SD-NEXT:    movi v4.2d, #00000000000000001460; CHECK-SD-NEXT:    movi v5.2d, #00000000000000001461; CHECK-SD-NEXT:    usdot v5.4s, v0.16b, v1.16b1462; CHECK-SD-NEXT:    usdot v4.4s, v2.16b, v3.16b1463; CHECK-SD-NEXT:    add v0.4s, v5.4s, v4.4s1464; CHECK-SD-NEXT:    addv s0, v0.4s1465; CHECK-SD-NEXT:    fmov w0, s01466; CHECK-SD-NEXT:    ret1467;1468; CHECK-GI-LABEL: test_usdot_v16i8_double:1469; CHECK-GI:       // %bb.0: // %entry1470; CHECK-GI-NEXT:    ushll v4.8h, v0.8b, #01471; CHECK-GI-NEXT:    ushll2 v0.8h, v0.16b, #01472; CHECK-GI-NEXT:    sshll v5.8h, v1.8b, #01473; CHECK-GI-NEXT:    sshll2 v1.8h, v1.16b, #01474; CHECK-GI-NEXT:    ushll v6.8h, v2.8b, #01475; CHECK-GI-NEXT:    ushll2 v2.8h, v2.16b, #01476; CHECK-GI-NEXT:    sshll v7.8h, v3.8b, #01477; CHECK-GI-NEXT:    sshll2 v3.8h, v3.16b, #01478; CHECK-GI-NEXT:    smull v16.4s, v4.4h, v5.4h1479; CHECK-GI-NEXT:    smull v17.4s, v0.4h, v1.4h1480; CHECK-GI-NEXT:    smull v18.4s, v6.4h, v7.4h1481; CHECK-GI-NEXT:    smull v19.4s, v2.4h, v3.4h1482; CHECK-GI-NEXT:    smlal2 v16.4s, v4.8h, v5.8h1483; CHECK-GI-NEXT:    smlal2 v17.4s, v0.8h, v1.8h1484; CHECK-GI-NEXT:    smlal2 v18.4s, v6.8h, v7.8h1485; CHECK-GI-NEXT:    smlal2 v19.4s, v2.8h, v3.8h1486; CHECK-GI-NEXT:    add v0.4s, v16.4s, v17.4s1487; CHECK-GI-NEXT:    add v1.4s, v18.4s, v19.4s1488; CHECK-GI-NEXT:    addv s0, v0.4s1489; CHECK-GI-NEXT:    addv s1, v1.4s1490; CHECK-GI-NEXT:    fmov w8, s01491; CHECK-GI-NEXT:    fmov w9, s11492; CHECK-GI-NEXT:    add w0, w8, w91493; CHECK-GI-NEXT:    ret1494entry:1495  %az = zext <16 x i8> %a to <16 x i32>1496  %bz = sext <16 x i8> %b to <16 x i32>1497  %m1 = mul nuw nsw <16 x i32> %az, %bz1498  %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m1)1499  %cz = zext <16 x i8> %c to <16 x i32>1500  %dz = sext <16 x i8> %d to <16 x i32>1501  %m2 = mul nuw nsw <16 x i32> %cz, %dz1502  %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m2)1503  %x = add i32 %r1, %r21504  ret i32 %x1505}1506 1507 1508define i32 @test_usdot_swapped_operands_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {1509; CHECK-SD-LABEL: test_usdot_swapped_operands_v16i8_double:1510; CHECK-SD:       // %bb.0: // %entry1511; CHECK-SD-NEXT:    movi v4.2d, #00000000000000001512; CHECK-SD-NEXT:    movi v5.2d, #00000000000000001513; CHECK-SD-NEXT:    usdot v5.4s, v1.16b, v0.16b1514; CHECK-SD-NEXT:    usdot v4.4s, v3.16b, v2.16b1515; CHECK-SD-NEXT:    add v0.4s, v5.4s, v4.4s1516; CHECK-SD-NEXT:    addv s0, v0.4s1517; CHECK-SD-NEXT:    fmov w0, s01518; CHECK-SD-NEXT:    ret1519;1520; CHECK-GI-LABEL: test_usdot_swapped_operands_v16i8_double:1521; CHECK-GI:       // %bb.0: // %entry1522; CHECK-GI-NEXT:    sshll v4.8h, v0.8b, #01523; CHECK-GI-NEXT:    sshll2 v0.8h, v0.16b, #01524; CHECK-GI-NEXT:    ushll v5.8h, v1.8b, #01525; CHECK-GI-NEXT:    ushll2 v1.8h, v1.16b, #01526; CHECK-GI-NEXT:    sshll v6.8h, v2.8b, #01527; CHECK-GI-NEXT:    sshll2 v2.8h, v2.16b, #01528; CHECK-GI-NEXT:    ushll v7.8h, v3.8b, #01529; CHECK-GI-NEXT:    ushll2 v3.8h, v3.16b, #01530; CHECK-GI-NEXT:    smull v16.4s, v4.4h, v5.4h1531; CHECK-GI-NEXT:    smull v17.4s, v0.4h, v1.4h1532; CHECK-GI-NEXT:    smull v18.4s, v6.4h, v7.4h1533; CHECK-GI-NEXT:    smull v19.4s, v2.4h, v3.4h1534; CHECK-GI-NEXT:    smlal2 v16.4s, v4.8h, v5.8h1535; CHECK-GI-NEXT:    smlal2 v17.4s, v0.8h, v1.8h1536; CHECK-GI-NEXT:    smlal2 v18.4s, v6.8h, v7.8h1537; CHECK-GI-NEXT:    smlal2 v19.4s, v2.8h, v3.8h1538; CHECK-GI-NEXT:    add v0.4s, v16.4s, v17.4s1539; CHECK-GI-NEXT:    add v1.4s, v18.4s, v19.4s1540; CHECK-GI-NEXT:    addv s0, v0.4s1541; CHECK-GI-NEXT:    addv s1, v1.4s1542; CHECK-GI-NEXT:    fmov w8, s01543; CHECK-GI-NEXT:    fmov w9, s11544; CHECK-GI-NEXT:    add w0, w8, w91545; CHECK-GI-NEXT:    ret1546entry:1547  %az = sext <16 x i8> %a to <16 x i32>1548  %bz = zext <16 x i8> %b to <16 x i32>1549  %m1 = mul nuw nsw <16 x i32> %az, %bz1550  %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m1)1551  %cz = sext <16 x i8> %c to <16 x i32>1552  %dz = zext <16 x i8> %d to <16 x i32>1553  %m2 = mul nuw nsw <16 x i32> %cz, %dz1554  %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m2)1555  %x = add i32 %r1, %r21556  ret i32 %x1557}1558 1559define i32 @test_udot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {1560; CHECK-SD-LABEL: test_udot_v24i8:1561; CHECK-SD:       // %bb.0: // %entry1562; CHECK-SD-NEXT:    movi v0.2d, #00000000000000001563; CHECK-SD-NEXT:    movi v1.2d, #00000000000000001564; CHECK-SD-NEXT:    ldr q2, [x0]1565; CHECK-SD-NEXT:    ldr q3, [x1]1566; CHECK-SD-NEXT:    ldr d4, [x0, #16]1567; CHECK-SD-NEXT:    ldr d5, [x1, #16]1568; CHECK-SD-NEXT:    udot v1.2s, v5.8b, v4.8b1569; CHECK-SD-NEXT:    udot v0.4s, v3.16b, v2.16b1570; CHECK-SD-NEXT:    addp v1.2s, v1.2s, v1.2s1571; CHECK-SD-NEXT:    addv s0, v0.4s1572; CHECK-SD-NEXT:    fmov w8, s11573; CHECK-SD-NEXT:    fmov w9, s01574; CHECK-SD-NEXT:    add w8, w9, w81575; CHECK-SD-NEXT:    add w0, w8, w21576; CHECK-SD-NEXT:    ret1577;1578; CHECK-GI-LABEL: test_udot_v24i8:1579; CHECK-GI:       // %bb.0: // %entry1580; CHECK-GI-NEXT:    movi v0.2d, #00000000000000001581; CHECK-GI-NEXT:    movi v1.2d, #00000000000000001582; CHECK-GI-NEXT:    ldr q2, [x0]1583; CHECK-GI-NEXT:    ldr d3, [x0, #16]1584; CHECK-GI-NEXT:    ldr q4, [x1]1585; CHECK-GI-NEXT:    ldr d5, [x1, #16]1586; CHECK-GI-NEXT:    udot v1.4s, v4.16b, v2.16b1587; CHECK-GI-NEXT:    udot v0.4s, v5.16b, v3.16b1588; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s1589; CHECK-GI-NEXT:    addv s0, v0.4s1590; CHECK-GI-NEXT:    fmov w8, s01591; CHECK-GI-NEXT:    add w0, w8, w21592; CHECK-GI-NEXT:    ret1593entry:1594  %0 = load <24 x i8>, ptr %a1595  %1 = zext <24 x i8> %0 to <24 x i32>1596  %2 = load <24 x i8>, ptr %b1597  %3 = zext <24 x i8> %2 to <24 x i32>1598  %4 = mul nuw nsw <24 x i32> %3, %11599  %5 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %4)1600  %op.extra = add i32 %5, %sum1601  ret i32 %op.extra1602}1603 1604define i32 @test_udot_v24i8_nomla(ptr nocapture readonly %a1) {1605; CHECK-SD-LABEL: test_udot_v24i8_nomla:1606; CHECK-SD:       // %bb.0: // %entry1607; CHECK-SD-NEXT:    movi v0.2d, #00000000000000001608; CHECK-SD-NEXT:    movi v1.8b, #11609; CHECK-SD-NEXT:    ldr q4, [x0]1610; CHECK-SD-NEXT:    movi v2.2d, #00000000000000001611; CHECK-SD-NEXT:    movi v3.16b, #11612; CHECK-SD-NEXT:    ldr d5, [x0, #16]1613; CHECK-SD-NEXT:    udot v2.2s, v5.8b, v1.8b1614; CHECK-SD-NEXT:    udot v0.4s, v4.16b, v3.16b1615; CHECK-SD-NEXT:    addp v1.2s, v2.2s, v2.2s1616; CHECK-SD-NEXT:    addv s0, v0.4s1617; CHECK-SD-NEXT:    fmov w8, s11618; CHECK-SD-NEXT:    fmov w9, s01619; CHECK-SD-NEXT:    add w0, w9, w81620; CHECK-SD-NEXT:    ret1621;1622; CHECK-GI-LABEL: test_udot_v24i8_nomla:1623; CHECK-GI:       // %bb.0: // %entry1624; CHECK-GI-NEXT:    movi v0.8b, #11625; CHECK-GI-NEXT:    movi v1.8b, #11626; CHECK-GI-NEXT:    ldr q4, [x0]1627; CHECK-GI-NEXT:    movi v2.2d, #00000000000000001628; CHECK-GI-NEXT:    movi v3.2d, #00000000000000001629; CHECK-GI-NEXT:    ldr d5, [x0, #16]1630; CHECK-GI-NEXT:    mov v1.d[1], v0.d[0]1631; CHECK-GI-NEXT:    udot v2.4s, v5.16b, v0.16b1632; CHECK-GI-NEXT:    udot v3.4s, v4.16b, v1.16b1633; CHECK-GI-NEXT:    add v0.4s, v3.4s, v2.4s1634; CHECK-GI-NEXT:    addv s0, v0.4s1635; CHECK-GI-NEXT:    fmov w0, s01636; CHECK-GI-NEXT:    ret1637entry:1638  %0 = load <24 x i8>, ptr %a11639  %1 = zext <24 x i8> %0 to <24 x i32>1640  %2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %1)1641  ret i32 %21642}1643define i32 @test_sdot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {1644; CHECK-SD-LABEL: test_sdot_v24i8:1645; CHECK-SD:       // %bb.0: // %entry1646; CHECK-SD-NEXT:    movi v0.2d, #00000000000000001647; CHECK-SD-NEXT:    movi v1.2d, #00000000000000001648; CHECK-SD-NEXT:    ldr q2, [x0]1649; CHECK-SD-NEXT:    ldr q3, [x1]1650; CHECK-SD-NEXT:    ldr d4, [x0, #16]1651; CHECK-SD-NEXT:    ldr d5, [x1, #16]1652; CHECK-SD-NEXT:    sdot v1.2s, v5.8b, v4.8b1653; CHECK-SD-NEXT:    sdot v0.4s, v3.16b, v2.16b1654; CHECK-SD-NEXT:    addp v1.2s, v1.2s, v1.2s1655; CHECK-SD-NEXT:    addv s0, v0.4s1656; CHECK-SD-NEXT:    fmov w8, s11657; CHECK-SD-NEXT:    fmov w9, s01658; CHECK-SD-NEXT:    add w8, w9, w81659; CHECK-SD-NEXT:    add w0, w8, w21660; CHECK-SD-NEXT:    ret1661;1662; CHECK-GI-LABEL: test_sdot_v24i8:1663; CHECK-GI:       // %bb.0: // %entry1664; CHECK-GI-NEXT:    movi v0.2d, #00000000000000001665; CHECK-GI-NEXT:    movi v1.2d, #00000000000000001666; CHECK-GI-NEXT:    ldr q2, [x0]1667; CHECK-GI-NEXT:    ldr d3, [x0, #16]1668; CHECK-GI-NEXT:    ldr q4, [x1]1669; CHECK-GI-NEXT:    ldr d5, [x1, #16]1670; CHECK-GI-NEXT:    sdot v1.4s, v4.16b, v2.16b1671; CHECK-GI-NEXT:    sdot v0.4s, v5.16b, v3.16b1672; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s1673; CHECK-GI-NEXT:    addv s0, v0.4s1674; CHECK-GI-NEXT:    fmov w8, s01675; CHECK-GI-NEXT:    add w0, w8, w21676; CHECK-GI-NEXT:    ret1677entry:1678  %0 = load <24 x i8>, ptr %a1679  %1 = sext <24 x i8> %0 to <24 x i32>1680  %2 = load <24 x i8>, ptr %b1681  %3 = sext <24 x i8> %2 to <24 x i32>1682  %4 = mul nsw <24 x i32> %3, %11683  %5 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %4)1684  %op.extra = add nsw i32 %5, %sum1685  ret i32 %op.extra1686}1687 1688define i32 @test_sdot_v24i8_double(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) {1689; CHECK-SD-LABEL: test_sdot_v24i8_double:1690; CHECK-SD:       // %bb.0: // %entry1691; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill1692; CHECK-SD-NEXT:    .cfi_def_cfa_offset 161693; CHECK-SD-NEXT:    .cfi_offset w29, -161694; CHECK-SD-NEXT:    fmov s0, w01695; CHECK-SD-NEXT:    ldr b1, [sp, #144]1696; CHECK-SD-NEXT:    add x10, sp, #1521697; CHECK-SD-NEXT:    add x9, sp, #1601698; CHECK-SD-NEXT:    add x8, sp, #1681699; CHECK-SD-NEXT:    ldr b2, [sp, #272]1700; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x10]1701; CHECK-SD-NEXT:    add x11, sp, #2801702; CHECK-SD-NEXT:    ldr b3, [sp, #80]1703; CHECK-SD-NEXT:    mov v0.b[1], w11704; CHECK-SD-NEXT:    ldr b4, [sp, #528]1705; CHECK-SD-NEXT:    add x10, sp, #881706; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x11]1707; CHECK-SD-NEXT:    add x11, sp, #5361708; CHECK-SD-NEXT:    ldr b5, [sp, #336]1709; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x9]1710; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x10]1711; CHECK-SD-NEXT:    add x10, sp, #3441712; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x11]1713; CHECK-SD-NEXT:    add x11, sp, #1761714; CHECK-SD-NEXT:    ldr b6, [sp, #656]1715; CHECK-SD-NEXT:    mov v0.b[2], w21716; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x10]1717; CHECK-SD-NEXT:    ldr b7, [sp, #464]1718; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]1719; CHECK-SD-NEXT:    add x12, sp, #6641720; CHECK-SD-NEXT:    add x9, sp, #4721721; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x12]1722; CHECK-SD-NEXT:    add x8, sp, #961723; CHECK-SD-NEXT:    add x10, sp, #1841724; CHECK-SD-NEXT:    add x12, sp, #2881725; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x9]1726; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x8]1727; CHECK-SD-NEXT:    mov v0.b[3], w31728; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x11]1729; CHECK-SD-NEXT:    add x8, sp, #3521730; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x12]1731; CHECK-SD-NEXT:    add x13, sp, #5441732; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x8]1733; CHECK-SD-NEXT:    add x8, sp, #6721734; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x13]1735; CHECK-SD-NEXT:    add x9, sp, #1921736; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x10]1737; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]1738; CHECK-SD-NEXT:    add x8, sp, #4801739; CHECK-SD-NEXT:    mov v0.b[4], w41740; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x8]1741; CHECK-SD-NEXT:    add x8, sp, #2961742; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x8]1743; CHECK-SD-NEXT:    add x8, sp, #5521744; CHECK-SD-NEXT:    add x12, sp, #2001745; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x9]1746; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x8]1747; CHECK-SD-NEXT:    add x8, sp, #3601748; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x8]1749; CHECK-SD-NEXT:    add x8, sp, #1041750; CHECK-SD-NEXT:    add x9, sp, #5601751; CHECK-SD-NEXT:    mov v0.b[5], w51752; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x8]1753; CHECK-SD-NEXT:    add x8, sp, #3681754; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x12]1755; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x9]1756; CHECK-SD-NEXT:    add x13, sp, #2081757; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x8]1758; CHECK-SD-NEXT:    add x12, sp, #3041759; CHECK-SD-NEXT:    add x8, sp, #5681760; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x12]1761; CHECK-SD-NEXT:    add x12, sp, #161762; CHECK-SD-NEXT:    add x17, sp, #3761763; CHECK-SD-NEXT:    mov v0.b[6], w61764; CHECK-SD-NEXT:    ld1 { v1.b }[8], [x13]1765; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x8]1766; CHECK-SD-NEXT:    add x14, sp, #2161767; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x17]1768; CHECK-SD-NEXT:    add x13, sp, #5761769; CHECK-SD-NEXT:    add x11, sp, #2241770; CHECK-SD-NEXT:    add x10, sp, #2321771; CHECK-SD-NEXT:    add x15, sp, #2401772; CHECK-SD-NEXT:    ld1 { v1.b }[9], [x14]1773; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x13]1774; CHECK-SD-NEXT:    add x13, sp, #3841775; CHECK-SD-NEXT:    mov v0.b[7], w71776; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x13]1777; CHECK-SD-NEXT:    add x13, sp, #1121778; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x13]1779; CHECK-SD-NEXT:    add x13, sp, #321780; CHECK-SD-NEXT:    add x14, sp, #5841781; CHECK-SD-NEXT:    ld1 { v1.b }[10], [x11]1782; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x14]1783; CHECK-SD-NEXT:    add x11, sp, #3121784; CHECK-SD-NEXT:    add x14, sp, #401785; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x11]1786; CHECK-SD-NEXT:    add x11, sp, #5921787; CHECK-SD-NEXT:    ld1 { v0.b }[8], [x12]1788; CHECK-SD-NEXT:    add x12, sp, #241789; CHECK-SD-NEXT:    add x16, sp, #2481790; CHECK-SD-NEXT:    ld1 { v1.b }[11], [x10]1791; CHECK-SD-NEXT:    ld1 { v4.b }[8], [x11]1792; CHECK-SD-NEXT:    add x11, sp, #4001793; CHECK-SD-NEXT:    add x9, sp, #2561794; CHECK-SD-NEXT:    add x8, sp, #2641795; CHECK-SD-NEXT:    add x10, sp, #721796; CHECK-SD-NEXT:    ld1 { v0.b }[9], [x12]1797; CHECK-SD-NEXT:    add x12, sp, #3921798; CHECK-SD-NEXT:    movi v16.2d, #00000000000000001799; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x12]1800; CHECK-SD-NEXT:    add x12, sp, #481801; CHECK-SD-NEXT:    ld1 { v1.b }[12], [x15]1802; CHECK-SD-NEXT:    add x15, sp, #1201803; CHECK-SD-NEXT:    movi v17.2d, #00000000000000001804; CHECK-SD-NEXT:    movi v18.2d, #00000000000000001805; CHECK-SD-NEXT:    ld1 { v0.b }[10], [x13]1806; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x15]1807; CHECK-SD-NEXT:    add x15, sp, #4081808; CHECK-SD-NEXT:    ld1 { v5.b }[8], [x11]1809; CHECK-SD-NEXT:    add x13, sp, #561810; CHECK-SD-NEXT:    ld1 { v1.b }[13], [x16]1811; CHECK-SD-NEXT:    add x11, sp, #641812; CHECK-SD-NEXT:    add x16, sp, #6161813; CHECK-SD-NEXT:    movi v19.2d, #00000000000000001814; CHECK-SD-NEXT:    ld1 { v0.b }[11], [x14]1815; CHECK-SD-NEXT:    add x14, sp, #6001816; CHECK-SD-NEXT:    ld1 { v4.b }[9], [x14]1817; CHECK-SD-NEXT:    ld1 { v5.b }[9], [x15]1818; CHECK-SD-NEXT:    add x15, sp, #6081819; CHECK-SD-NEXT:    ld1 { v1.b }[14], [x9]1820; CHECK-SD-NEXT:    add x9, sp, #4881821; CHECK-SD-NEXT:    add x14, sp, #3201822; CHECK-SD-NEXT:    ld1 { v0.b }[12], [x12]1823; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]1824; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x14]1825; CHECK-SD-NEXT:    ld1 { v4.b }[10], [x15]1826; CHECK-SD-NEXT:    add x14, sp, #6241827; CHECK-SD-NEXT:    add x9, sp, #6881828; CHECK-SD-NEXT:    ld1 { v1.b }[15], [x8]1829; CHECK-SD-NEXT:    add x8, sp, #4321830; CHECK-SD-NEXT:    add x12, sp, #3281831; CHECK-SD-NEXT:    ld1 { v0.b }[13], [x13]1832; CHECK-SD-NEXT:    add x13, sp, #4161833; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x12]1834; CHECK-SD-NEXT:    ld1 { v5.b }[10], [x13]1835; CHECK-SD-NEXT:    ld1 { v4.b }[11], [x16]1836; CHECK-SD-NEXT:    add x16, sp, #6801837; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x16]1838; CHECK-SD-NEXT:    add x13, sp, #6321839; CHECK-SD-NEXT:    add x12, sp, #5041840; CHECK-SD-NEXT:    ld1 { v0.b }[14], [x11]1841; CHECK-SD-NEXT:    add x11, sp, #4241842; CHECK-SD-NEXT:    add x15, sp, #1281843; CHECK-SD-NEXT:    ld1 { v5.b }[11], [x11]1844; CHECK-SD-NEXT:    ld1 { v4.b }[12], [x14]1845; CHECK-SD-NEXT:    add x11, sp, #6961846; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x9]1847; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x15]1848; CHECK-SD-NEXT:    add x9, sp, #6401849; CHECK-SD-NEXT:    ld1 { v0.b }[15], [x10]1850; CHECK-SD-NEXT:    add x10, sp, #4961851; CHECK-SD-NEXT:    ld1 { v5.b }[12], [x8]1852; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]1853; CHECK-SD-NEXT:    ld1 { v4.b }[13], [x13]1854; CHECK-SD-NEXT:    add x10, sp, #4401855; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x11]1856; CHECK-SD-NEXT:    add x11, sp, #5121857; CHECK-SD-NEXT:    add x8, sp, #1361858; CHECK-SD-NEXT:    sdot v17.4s, v0.16b, v1.16b1859; CHECK-SD-NEXT:    ld1 { v5.b }[13], [x10]1860; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x12]1861; CHECK-SD-NEXT:    ld1 { v4.b }[14], [x9]1862; CHECK-SD-NEXT:    add x9, sp, #4481863; CHECK-SD-NEXT:    add x10, sp, #7041864; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x8]1865; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x10]1866; CHECK-SD-NEXT:    add x8, sp, #6481867; CHECK-SD-NEXT:    add x10, sp, #5201868; CHECK-SD-NEXT:    ld1 { v5.b }[14], [x9]1869; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x11]1870; CHECK-SD-NEXT:    ld1 { v4.b }[15], [x8]1871; CHECK-SD-NEXT:    add x8, sp, #4561872; CHECK-SD-NEXT:    add x9, sp, #7121873; CHECK-SD-NEXT:    sdot v19.2s, v3.8b, v2.8b1874; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x9]1875; CHECK-SD-NEXT:    addv s0, v17.4s1876; CHECK-SD-NEXT:    ld1 { v5.b }[15], [x8]1877; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x10]1878; CHECK-SD-NEXT:    addp v1.2s, v19.2s, v19.2s1879; CHECK-SD-NEXT:    fmov w8, s01880; CHECK-SD-NEXT:    sdot v16.4s, v5.16b, v4.16b1881; CHECK-SD-NEXT:    sdot v18.2s, v7.8b, v6.8b1882; CHECK-SD-NEXT:    fmov w9, s11883; CHECK-SD-NEXT:    addv s2, v16.4s1884; CHECK-SD-NEXT:    addp v3.2s, v18.2s, v18.2s1885; CHECK-SD-NEXT:    add w8, w8, w91886; CHECK-SD-NEXT:    fmov w10, s21887; CHECK-SD-NEXT:    fmov w11, s31888; CHECK-SD-NEXT:    add w9, w10, w111889; CHECK-SD-NEXT:    add w0, w8, w91890; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload1891; CHECK-SD-NEXT:    ret1892;1893; CHECK-GI-LABEL: test_sdot_v24i8_double:1894; CHECK-GI:       // %bb.0: // %entry1895; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill1896; CHECK-GI-NEXT:    .cfi_def_cfa_offset 161897; CHECK-GI-NEXT:    .cfi_offset w29, -161898; CHECK-GI-NEXT:    ldr w8, [sp, #80]1899; CHECK-GI-NEXT:    ldr w9, [sp, #88]1900; CHECK-GI-NEXT:    fmov s1, w01901; CHECK-GI-NEXT:    ldr w11, [sp, #336]1902; CHECK-GI-NEXT:    ldr w10, [sp, #280]1903; CHECK-GI-NEXT:    movi v16.2d, #00000000000000001904; CHECK-GI-NEXT:    fmov s0, w81905; CHECK-GI-NEXT:    ldr w8, [sp, #96]1906; CHECK-GI-NEXT:    ldr w12, [sp, #152]1907; CHECK-GI-NEXT:    mov v1.b[1], w11908; CHECK-GI-NEXT:    fmov s4, w111909; CHECK-GI-NEXT:    ldr w11, [sp, #584]1910; CHECK-GI-NEXT:    movi v17.2d, #00000000000000001911; CHECK-GI-NEXT:    movi v18.2d, #00000000000000001912; CHECK-GI-NEXT:    movi v19.2d, #00000000000000001913; CHECK-GI-NEXT:    mov v0.b[1], w91914; CHECK-GI-NEXT:    ldr w9, [sp, #272]1915; CHECK-GI-NEXT:    fmov s2, w91916; CHECK-GI-NEXT:    ldr w9, [sp, #144]1917; CHECK-GI-NEXT:    mov v1.b[2], w21918; CHECK-GI-NEXT:    mov v0.b[2], w81919; CHECK-GI-NEXT:    ldr w8, [sp, #528]1920; CHECK-GI-NEXT:    fmov s3, w91921; CHECK-GI-NEXT:    mov v2.b[1], w101922; CHECK-GI-NEXT:    ldr w9, [sp, #344]1923; CHECK-GI-NEXT:    ldr w10, [sp, #536]1924; CHECK-GI-NEXT:    fmov s5, w81925; CHECK-GI-NEXT:    ldr w8, [sp, #288]1926; CHECK-GI-NEXT:    mov v1.b[3], w31927; CHECK-GI-NEXT:    mov v3.b[1], w121928; CHECK-GI-NEXT:    mov v4.b[1], w91929; CHECK-GI-NEXT:    ldr w9, [sp, #160]1930; CHECK-GI-NEXT:    mov v5.b[1], w101931; CHECK-GI-NEXT:    mov v2.b[2], w81932; CHECK-GI-NEXT:    ldr w8, [sp, #104]1933; CHECK-GI-NEXT:    ldr w10, [sp, #352]1934; CHECK-GI-NEXT:    mov v1.b[4], w41935; CHECK-GI-NEXT:    mov v3.b[2], w91936; CHECK-GI-NEXT:    ldr w9, [sp, #544]1937; CHECK-GI-NEXT:    mov v0.b[3], w81938; CHECK-GI-NEXT:    ldr w8, [sp, #296]1939; CHECK-GI-NEXT:    mov v4.b[2], w101940; CHECK-GI-NEXT:    ldr w10, [sp, #360]1941; CHECK-GI-NEXT:    mov v5.b[2], w91942; CHECK-GI-NEXT:    ldr w9, [sp, #168]1943; CHECK-GI-NEXT:    mov v2.b[3], w81944; CHECK-GI-NEXT:    ldr w8, [sp, #112]1945; CHECK-GI-NEXT:    mov v1.b[5], w51946; CHECK-GI-NEXT:    mov v3.b[3], w91947; CHECK-GI-NEXT:    ldr w9, [sp, #552]1948; CHECK-GI-NEXT:    mov v0.b[4], w81949; CHECK-GI-NEXT:    ldr w8, [sp, #304]1950; CHECK-GI-NEXT:    mov v4.b[3], w101951; CHECK-GI-NEXT:    mov v5.b[3], w91952; CHECK-GI-NEXT:    ldr w9, [sp, #176]1953; CHECK-GI-NEXT:    ldr w10, [sp, #368]1954; CHECK-GI-NEXT:    mov v2.b[4], w81955; CHECK-GI-NEXT:    ldr w8, [sp, #120]1956; CHECK-GI-NEXT:    mov v1.b[6], w61957; CHECK-GI-NEXT:    mov v3.b[4], w91958; CHECK-GI-NEXT:    ldr w9, [sp, #560]1959; CHECK-GI-NEXT:    mov v0.b[5], w81960; CHECK-GI-NEXT:    ldr w8, [sp, #312]1961; CHECK-GI-NEXT:    mov v4.b[4], w101962; CHECK-GI-NEXT:    mov v5.b[4], w91963; CHECK-GI-NEXT:    ldr w9, [sp, #184]1964; CHECK-GI-NEXT:    ldr w10, [sp, #376]1965; CHECK-GI-NEXT:    mov v2.b[5], w81966; CHECK-GI-NEXT:    ldr w8, [sp, #128]1967; CHECK-GI-NEXT:    mov v1.b[7], w71968; CHECK-GI-NEXT:    mov v3.b[5], w91969; CHECK-GI-NEXT:    ldr w9, [sp, #568]1970; CHECK-GI-NEXT:    mov v0.b[6], w81971; CHECK-GI-NEXT:    ldr w8, [sp, #320]1972; CHECK-GI-NEXT:    mov v4.b[5], w101973; CHECK-GI-NEXT:    mov v5.b[5], w91974; CHECK-GI-NEXT:    ldr w9, [sp, #192]1975; CHECK-GI-NEXT:    ldr w10, [sp, #384]1976; CHECK-GI-NEXT:    mov v2.b[6], w81977; CHECK-GI-NEXT:    ldr w8, [sp, #136]1978; CHECK-GI-NEXT:    mov v3.b[6], w91979; CHECK-GI-NEXT:    ldr w9, [sp, #576]1980; CHECK-GI-NEXT:    mov v0.b[7], w81981; CHECK-GI-NEXT:    ldr w8, [sp, #328]1982; CHECK-GI-NEXT:    mov v4.b[6], w101983; CHECK-GI-NEXT:    ldr w10, [sp, #200]1984; CHECK-GI-NEXT:    mov v5.b[6], w91985; CHECK-GI-NEXT:    ldr w9, [sp, #392]1986; CHECK-GI-NEXT:    mov v2.b[7], w81987; CHECK-GI-NEXT:    ldr w8, [sp, #464]1988; CHECK-GI-NEXT:    mov v3.b[7], w101989; CHECK-GI-NEXT:    ldr w10, [sp, #16]1990; CHECK-GI-NEXT:    fmov s6, w81991; CHECK-GI-NEXT:    ldr w8, [sp, #208]1992; CHECK-GI-NEXT:    mov v4.b[7], w91993; CHECK-GI-NEXT:    mov v1.b[8], w101994; CHECK-GI-NEXT:    ldr w10, [sp, #656]1995; CHECK-GI-NEXT:    ldr w9, [sp, #472]1996; CHECK-GI-NEXT:    mov v5.b[7], w111997; CHECK-GI-NEXT:    ldr w11, [sp, #400]1998; CHECK-GI-NEXT:    fmov d0, d01999; CHECK-GI-NEXT:    fmov s7, w102000; CHECK-GI-NEXT:    mov v6.b[1], w92001; CHECK-GI-NEXT:    ldr w9, [sp, #592]2002; CHECK-GI-NEXT:    mov v3.b[8], w82003; CHECK-GI-NEXT:    ldr w10, [sp, #664]2004; CHECK-GI-NEXT:    ldr w8, [sp, #24]2005; CHECK-GI-NEXT:    mov v4.b[8], w112006; CHECK-GI-NEXT:    ldr w11, [sp, #216]2007; CHECK-GI-NEXT:    fmov d2, d22008; CHECK-GI-NEXT:    mov v5.b[8], w92009; CHECK-GI-NEXT:    ldr w9, [sp, #480]2010; CHECK-GI-NEXT:    mov v7.b[1], w102011; CHECK-GI-NEXT:    mov v1.b[9], w82012; CHECK-GI-NEXT:    ldr w8, [sp, #408]2013; CHECK-GI-NEXT:    ldr w10, [sp, #600]2014; CHECK-GI-NEXT:    mov v3.b[9], w112015; CHECK-GI-NEXT:    mov v6.b[2], w92016; CHECK-GI-NEXT:    ldr w9, [sp, #672]2017; CHECK-GI-NEXT:    ldr w11, [sp, #32]2018; CHECK-GI-NEXT:    mov v4.b[9], w82019; CHECK-GI-NEXT:    ldr w8, [sp, #224]2020; CHECK-GI-NEXT:    mov v5.b[9], w102021; CHECK-GI-NEXT:    ldr w10, [sp, #488]2022; CHECK-GI-NEXT:    mov v7.b[2], w92023; CHECK-GI-NEXT:    mov v1.b[10], w112024; CHECK-GI-NEXT:    ldr w9, [sp, #416]2025; CHECK-GI-NEXT:    ldr w11, [sp, #608]2026; CHECK-GI-NEXT:    mov v3.b[10], w82027; CHECK-GI-NEXT:    mov v6.b[3], w102028; CHECK-GI-NEXT:    ldr w10, [sp, #680]2029; CHECK-GI-NEXT:    ldr w8, [sp, #40]2030; CHECK-GI-NEXT:    mov v4.b[10], w92031; CHECK-GI-NEXT:    ldr w9, [sp, #232]2032; CHECK-GI-NEXT:    mov v5.b[10], w112033; CHECK-GI-NEXT:    ldr w11, [sp, #496]2034; CHECK-GI-NEXT:    mov v7.b[3], w102035; CHECK-GI-NEXT:    mov v1.b[11], w82036; CHECK-GI-NEXT:    ldr w8, [sp, #424]2037; CHECK-GI-NEXT:    ldr w10, [sp, #616]2038; CHECK-GI-NEXT:    mov v3.b[11], w92039; CHECK-GI-NEXT:    mov v6.b[4], w112040; CHECK-GI-NEXT:    ldr w11, [sp, #688]2041; CHECK-GI-NEXT:    ldr w9, [sp, #48]2042; CHECK-GI-NEXT:    mov v4.b[11], w82043; CHECK-GI-NEXT:    ldr w8, [sp, #240]2044; CHECK-GI-NEXT:    mov v5.b[11], w102045; CHECK-GI-NEXT:    ldr w10, [sp, #504]2046; CHECK-GI-NEXT:    mov v7.b[4], w112047; CHECK-GI-NEXT:    mov v1.b[12], w92048; CHECK-GI-NEXT:    ldr w9, [sp, #432]2049; CHECK-GI-NEXT:    ldr w11, [sp, #624]2050; CHECK-GI-NEXT:    mov v3.b[12], w82051; CHECK-GI-NEXT:    mov v6.b[5], w102052; CHECK-GI-NEXT:    ldr w10, [sp, #696]2053; CHECK-GI-NEXT:    ldr w8, [sp, #56]2054; CHECK-GI-NEXT:    mov v4.b[12], w92055; CHECK-GI-NEXT:    ldr w9, [sp, #248]2056; CHECK-GI-NEXT:    mov v5.b[12], w112057; CHECK-GI-NEXT:    ldr w11, [sp, #512]2058; CHECK-GI-NEXT:    mov v7.b[5], w102059; CHECK-GI-NEXT:    mov v1.b[13], w82060; CHECK-GI-NEXT:    ldr w8, [sp, #440]2061; CHECK-GI-NEXT:    ldr w10, [sp, #632]2062; CHECK-GI-NEXT:    mov v3.b[13], w92063; CHECK-GI-NEXT:    mov v6.b[6], w112064; CHECK-GI-NEXT:    ldr w11, [sp, #704]2065; CHECK-GI-NEXT:    ldr w9, [sp, #64]2066; CHECK-GI-NEXT:    mov v4.b[13], w82067; CHECK-GI-NEXT:    ldr w8, [sp, #256]2068; CHECK-GI-NEXT:    mov v5.b[13], w102069; CHECK-GI-NEXT:    ldr w10, [sp, #520]2070; CHECK-GI-NEXT:    mov v7.b[6], w112071; CHECK-GI-NEXT:    mov v1.b[14], w92072; CHECK-GI-NEXT:    ldr w9, [sp, #448]2073; CHECK-GI-NEXT:    ldr w11, [sp, #640]2074; CHECK-GI-NEXT:    mov v3.b[14], w82075; CHECK-GI-NEXT:    mov v6.b[7], w102076; CHECK-GI-NEXT:    ldr w10, [sp, #712]2077; CHECK-GI-NEXT:    ldr w8, [sp, #72]2078; CHECK-GI-NEXT:    mov v4.b[14], w92079; CHECK-GI-NEXT:    ldr w9, [sp, #264]2080; CHECK-GI-NEXT:    mov v5.b[14], w112081; CHECK-GI-NEXT:    mov v7.b[7], w102082; CHECK-GI-NEXT:    sdot v18.4s, v0.16b, v2.16b2083; CHECK-GI-NEXT:    mov v1.b[15], w82084; CHECK-GI-NEXT:    ldr w8, [sp, #456]2085; CHECK-GI-NEXT:    mov v3.b[15], w92086; CHECK-GI-NEXT:    ldr w9, [sp, #648]2087; CHECK-GI-NEXT:    fmov d6, d62088; CHECK-GI-NEXT:    mov v4.b[15], w82089; CHECK-GI-NEXT:    mov v5.b[15], w92090; CHECK-GI-NEXT:    fmov d7, d72091; CHECK-GI-NEXT:    sdot v17.4s, v1.16b, v3.16b2092; CHECK-GI-NEXT:    sdot v19.4s, v4.16b, v5.16b2093; CHECK-GI-NEXT:    sdot v16.4s, v6.16b, v7.16b2094; CHECK-GI-NEXT:    add v0.4s, v17.4s, v18.4s2095; CHECK-GI-NEXT:    add v1.4s, v19.4s, v16.4s2096; CHECK-GI-NEXT:    addv s0, v0.4s2097; CHECK-GI-NEXT:    addv s1, v1.4s2098; CHECK-GI-NEXT:    fmov w8, s02099; CHECK-GI-NEXT:    fmov w9, s12100; CHECK-GI-NEXT:    add w0, w8, w92101; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload2102; CHECK-GI-NEXT:    ret2103entry:2104  %az = sext <24 x i8> %a to <24 x i32>2105  %bz = sext <24 x i8> %b to <24 x i32>2106  %m1 = mul nuw nsw <24 x i32> %az, %bz2107  %r1 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m1)2108  %cz = sext <24 x i8> %c to <24 x i32>2109  %dz = sext <24 x i8> %d to <24 x i32>2110  %m2 = mul nuw nsw <24 x i32> %cz, %dz2111  %r2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m2)2112  %x = add i32 %r1, %r22113  ret i32 %x2114}2115 2116define i32 @test_sdot_v24i8_double_nomla(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) {2117; CHECK-SD-LABEL: test_sdot_v24i8_double_nomla:2118; CHECK-SD:       // %bb.0: // %entry2119; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill2120; CHECK-SD-NEXT:    .cfi_def_cfa_offset 162121; CHECK-SD-NEXT:    .cfi_offset w29, -162122; CHECK-SD-NEXT:    fmov s0, w02123; CHECK-SD-NEXT:    ldr b1, [sp, #336]2124; CHECK-SD-NEXT:    add x8, sp, #3442125; CHECK-SD-NEXT:    add x9, sp, #4002126; CHECK-SD-NEXT:    ldr b2, [sp, #80]2127; CHECK-SD-NEXT:    ldr b3, [sp, #464]2128; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x8]2129; CHECK-SD-NEXT:    add x8, sp, #3522130; CHECK-SD-NEXT:    add x10, sp, #4082131; CHECK-SD-NEXT:    mov v0.b[1], w12132; CHECK-SD-NEXT:    add x11, sp, #4722133; CHECK-SD-NEXT:    add x12, sp, #4802134; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x11]2135; CHECK-SD-NEXT:    add x11, sp, #4162136; CHECK-SD-NEXT:    add x13, sp, #4882137; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x8]2138; CHECK-SD-NEXT:    add x8, sp, #3602139; CHECK-SD-NEXT:    add x14, sp, #4962140; CHECK-SD-NEXT:    movi v4.16b, #12141; CHECK-SD-NEXT:    movi v5.2d, #00000000000000002142; CHECK-SD-NEXT:    movi v6.2d, #00000000000000002143; CHECK-SD-NEXT:    mov v0.b[2], w22144; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x12]2145; CHECK-SD-NEXT:    add x12, sp, #4242146; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]2147; CHECK-SD-NEXT:    add x8, sp, #3682148; CHECK-SD-NEXT:    movi v7.2d, #00000000000000002149; CHECK-SD-NEXT:    movi v16.8b, #12150; CHECK-SD-NEXT:    movi v17.2d, #00000000000000002151; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x13]2152; CHECK-SD-NEXT:    add x13, sp, #4322153; CHECK-SD-NEXT:    mov v0.b[3], w32154; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x8]2155; CHECK-SD-NEXT:    add x8, sp, #3762156; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x14]2157; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x8]2158; CHECK-SD-NEXT:    add x8, sp, #3842159; CHECK-SD-NEXT:    mov v0.b[4], w42160; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x8]2161; CHECK-SD-NEXT:    add x8, sp, #3922162; CHECK-SD-NEXT:    mov v0.b[5], w52163; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x8]2164; CHECK-SD-NEXT:    add x8, sp, #162165; CHECK-SD-NEXT:    mov v0.b[6], w62166; CHECK-SD-NEXT:    ld1 { v1.b }[8], [x9]2167; CHECK-SD-NEXT:    add x9, sp, #882168; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x9]2169; CHECK-SD-NEXT:    add x9, sp, #402170; CHECK-SD-NEXT:    ld1 { v1.b }[9], [x10]2171; CHECK-SD-NEXT:    add x10, sp, #962172; CHECK-SD-NEXT:    mov v0.b[7], w72173; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x10]2174; CHECK-SD-NEXT:    add x10, sp, #562175; CHECK-SD-NEXT:    ld1 { v1.b }[10], [x11]2176; CHECK-SD-NEXT:    add x11, sp, #1042177; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x11]2178; CHECK-SD-NEXT:    add x11, sp, #722179; CHECK-SD-NEXT:    ld1 { v0.b }[8], [x8]2180; CHECK-SD-NEXT:    add x8, sp, #242181; CHECK-SD-NEXT:    ld1 { v1.b }[11], [x12]2182; CHECK-SD-NEXT:    add x12, sp, #1122183; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x12]2184; CHECK-SD-NEXT:    add x12, sp, #4402185; CHECK-SD-NEXT:    ld1 { v0.b }[9], [x8]2186; CHECK-SD-NEXT:    add x8, sp, #322187; CHECK-SD-NEXT:    ld1 { v1.b }[12], [x13]2188; CHECK-SD-NEXT:    add x13, sp, #5042189; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x13]2190; CHECK-SD-NEXT:    add x13, sp, #5122191; CHECK-SD-NEXT:    ld1 { v0.b }[10], [x8]2192; CHECK-SD-NEXT:    add x8, sp, #482193; CHECK-SD-NEXT:    ld1 { v1.b }[13], [x12]2194; CHECK-SD-NEXT:    add x12, sp, #4482195; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x13]2196; CHECK-SD-NEXT:    ld1 { v0.b }[11], [x9]2197; CHECK-SD-NEXT:    add x9, sp, #642198; CHECK-SD-NEXT:    ld1 { v1.b }[14], [x12]2199; CHECK-SD-NEXT:    ld1 { v0.b }[12], [x8]2200; CHECK-SD-NEXT:    add x8, sp, #1202201; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x8]2202; CHECK-SD-NEXT:    add x8, sp, #1282203; CHECK-SD-NEXT:    ld1 { v0.b }[13], [x10]2204; CHECK-SD-NEXT:    add x10, sp, #1362205; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x8]2206; CHECK-SD-NEXT:    add x8, sp, #4562207; CHECK-SD-NEXT:    ld1 { v1.b }[15], [x8]2208; CHECK-SD-NEXT:    ld1 { v0.b }[14], [x9]2209; CHECK-SD-NEXT:    add x9, sp, #5202210; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x10]2211; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x9]2212; CHECK-SD-NEXT:    sdot v5.4s, v1.16b, v4.16b2213; CHECK-SD-NEXT:    ld1 { v0.b }[15], [x11]2214; CHECK-SD-NEXT:    sdot v17.2s, v2.8b, v16.8b2215; CHECK-SD-NEXT:    sdot v7.2s, v3.8b, v16.8b2216; CHECK-SD-NEXT:    sdot v6.4s, v0.16b, v4.16b2217; CHECK-SD-NEXT:    addv s3, v5.4s2218; CHECK-SD-NEXT:    addp v1.2s, v17.2s, v17.2s2219; CHECK-SD-NEXT:    addp v2.2s, v7.2s, v7.2s2220; CHECK-SD-NEXT:    fmov w10, s32221; CHECK-SD-NEXT:    addv s0, v6.4s2222; CHECK-SD-NEXT:    fmov w9, s12223; CHECK-SD-NEXT:    fmov w11, s22224; CHECK-SD-NEXT:    fmov w8, s02225; CHECK-SD-NEXT:    add w8, w8, w92226; CHECK-SD-NEXT:    add w9, w10, w112227; CHECK-SD-NEXT:    add w0, w8, w92228; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload2229; CHECK-SD-NEXT:    ret2230;2231; CHECK-GI-LABEL: test_sdot_v24i8_double_nomla:2232; CHECK-GI:       // %bb.0: // %entry2233; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill2234; CHECK-GI-NEXT:    .cfi_def_cfa_offset 162235; CHECK-GI-NEXT:    .cfi_offset w29, -162236; CHECK-GI-NEXT:    ldr w9, [sp, #336]2237; CHECK-GI-NEXT:    ldr w8, [sp, #344]2238; CHECK-GI-NEXT:    fmov s0, w02239; CHECK-GI-NEXT:    ldr w10, [sp, #16]2240; CHECK-GI-NEXT:    ldr w11, [sp, #88]2241; CHECK-GI-NEXT:    movi v4.8b, #12242; CHECK-GI-NEXT:    fmov s1, w92243; CHECK-GI-NEXT:    ldr w9, [sp, #464]2244; CHECK-GI-NEXT:    ldr w12, [sp, #400]2245; CHECK-GI-NEXT:    mov v0.b[1], w12246; CHECK-GI-NEXT:    movi v5.8b, #12247; CHECK-GI-NEXT:    movi v6.8b, #12248; CHECK-GI-NEXT:    fmov s2, w92249; CHECK-GI-NEXT:    ldr w9, [sp, #96]2250; CHECK-GI-NEXT:    movi v7.2d, #00000000000000002251; CHECK-GI-NEXT:    mov v1.b[1], w82252; CHECK-GI-NEXT:    ldr w8, [sp, #352]2253; CHECK-GI-NEXT:    movi v16.2d, #00000000000000002254; CHECK-GI-NEXT:    movi v17.2d, #00000000000000002255; CHECK-GI-NEXT:    movi v18.2d, #00000000000000002256; CHECK-GI-NEXT:    mov v0.b[2], w22257; CHECK-GI-NEXT:    mov v5.d[1], v4.d[0]2258; CHECK-GI-NEXT:    mov v6.d[1], v4.d[0]2259; CHECK-GI-NEXT:    mov v1.b[2], w82260; CHECK-GI-NEXT:    ldr w8, [sp, #360]2261; CHECK-GI-NEXT:    mov v0.b[3], w32262; CHECK-GI-NEXT:    mov v1.b[3], w82263; CHECK-GI-NEXT:    ldr w8, [sp, #368]2264; CHECK-GI-NEXT:    mov v0.b[4], w42265; CHECK-GI-NEXT:    mov v1.b[4], w82266; CHECK-GI-NEXT:    ldr w8, [sp, #376]2267; CHECK-GI-NEXT:    mov v0.b[5], w52268; CHECK-GI-NEXT:    mov v1.b[5], w82269; CHECK-GI-NEXT:    ldr w8, [sp, #384]2270; CHECK-GI-NEXT:    mov v0.b[6], w62271; CHECK-GI-NEXT:    mov v1.b[6], w82272; CHECK-GI-NEXT:    ldr w8, [sp, #392]2273; CHECK-GI-NEXT:    mov v0.b[7], w72274; CHECK-GI-NEXT:    mov v1.b[7], w82275; CHECK-GI-NEXT:    ldr w8, [sp, #80]2276; CHECK-GI-NEXT:    fmov s3, w82277; CHECK-GI-NEXT:    ldr w8, [sp, #472]2278; CHECK-GI-NEXT:    mov v0.b[8], w102279; CHECK-GI-NEXT:    ldr w10, [sp, #408]2280; CHECK-GI-NEXT:    mov v1.b[8], w122281; CHECK-GI-NEXT:    mov v2.b[1], w82282; CHECK-GI-NEXT:    ldr w8, [sp, #24]2283; CHECK-GI-NEXT:    mov v3.b[1], w112284; CHECK-GI-NEXT:    ldr w11, [sp, #480]2285; CHECK-GI-NEXT:    mov v0.b[9], w82286; CHECK-GI-NEXT:    ldr w8, [sp, #32]2287; CHECK-GI-NEXT:    mov v1.b[9], w102288; CHECK-GI-NEXT:    mov v2.b[2], w112289; CHECK-GI-NEXT:    ldr w10, [sp, #416]2290; CHECK-GI-NEXT:    mov v3.b[2], w92291; CHECK-GI-NEXT:    ldr w9, [sp, #104]2292; CHECK-GI-NEXT:    ldr w11, [sp, #488]2293; CHECK-GI-NEXT:    mov v0.b[10], w82294; CHECK-GI-NEXT:    ldr w8, [sp, #40]2295; CHECK-GI-NEXT:    mov v1.b[10], w102296; CHECK-GI-NEXT:    mov v2.b[3], w112297; CHECK-GI-NEXT:    ldr w10, [sp, #424]2298; CHECK-GI-NEXT:    mov v3.b[3], w92299; CHECK-GI-NEXT:    ldr w9, [sp, #112]2300; CHECK-GI-NEXT:    ldr w11, [sp, #496]2301; CHECK-GI-NEXT:    mov v0.b[11], w82302; CHECK-GI-NEXT:    ldr w8, [sp, #48]2303; CHECK-GI-NEXT:    mov v1.b[11], w102304; CHECK-GI-NEXT:    mov v2.b[4], w112305; CHECK-GI-NEXT:    ldr w10, [sp, #432]2306; CHECK-GI-NEXT:    mov v3.b[4], w92307; CHECK-GI-NEXT:    ldr w9, [sp, #120]2308; CHECK-GI-NEXT:    ldr w11, [sp, #504]2309; CHECK-GI-NEXT:    mov v0.b[12], w82310; CHECK-GI-NEXT:    ldr w8, [sp, #56]2311; CHECK-GI-NEXT:    mov v1.b[12], w102312; CHECK-GI-NEXT:    mov v2.b[5], w112313; CHECK-GI-NEXT:    ldr w10, [sp, #440]2314; CHECK-GI-NEXT:    mov v3.b[5], w92315; CHECK-GI-NEXT:    ldr w9, [sp, #128]2316; CHECK-GI-NEXT:    ldr w11, [sp, #512]2317; CHECK-GI-NEXT:    mov v0.b[13], w82318; CHECK-GI-NEXT:    ldr w8, [sp, #64]2319; CHECK-GI-NEXT:    mov v1.b[13], w102320; CHECK-GI-NEXT:    mov v2.b[6], w112321; CHECK-GI-NEXT:    ldr w10, [sp, #448]2322; CHECK-GI-NEXT:    mov v3.b[6], w92323; CHECK-GI-NEXT:    ldr w9, [sp, #136]2324; CHECK-GI-NEXT:    ldr w11, [sp, #520]2325; CHECK-GI-NEXT:    mov v0.b[14], w82326; CHECK-GI-NEXT:    ldr w8, [sp, #72]2327; CHECK-GI-NEXT:    mov v1.b[14], w102328; CHECK-GI-NEXT:    mov v2.b[7], w112329; CHECK-GI-NEXT:    mov v3.b[7], w92330; CHECK-GI-NEXT:    ldr w9, [sp, #456]2331; CHECK-GI-NEXT:    mov v0.b[15], w82332; CHECK-GI-NEXT:    mov v1.b[15], w92333; CHECK-GI-NEXT:    fmov d2, d22334; CHECK-GI-NEXT:    fmov d3, d32335; CHECK-GI-NEXT:    sdot v16.4s, v0.16b, v5.16b2336; CHECK-GI-NEXT:    sdot v18.4s, v1.16b, v6.16b2337; CHECK-GI-NEXT:    sdot v7.4s, v2.16b, v4.16b2338; CHECK-GI-NEXT:    sdot v17.4s, v3.16b, v4.16b2339; CHECK-GI-NEXT:    add v1.4s, v18.4s, v7.4s2340; CHECK-GI-NEXT:    add v0.4s, v16.4s, v17.4s2341; CHECK-GI-NEXT:    addv s1, v1.4s2342; CHECK-GI-NEXT:    addv s0, v0.4s2343; CHECK-GI-NEXT:    fmov w9, s12344; CHECK-GI-NEXT:    fmov w8, s02345; CHECK-GI-NEXT:    add w0, w8, w92346; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload2347; CHECK-GI-NEXT:    ret2348entry:2349  %az = sext <24 x i8> %a to <24 x i32>2350  %r1 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %az)2351  %cz = sext <24 x i8> %c to <24 x i32>2352  %r2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %cz)2353  %x = add i32 %r1, %r22354  ret i32 %x2355}2356 2357define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {2358; CHECK-SD-LABEL: test_udot_v25i8:2359; CHECK-SD:       // %bb.0: // %entry2360; CHECK-SD-NEXT:    ldp q3, q0, [x1]2361; CHECK-SD-NEXT:    movi v5.2d, #00000000000000002362; CHECK-SD-NEXT:    ldp q2, q1, [x0]2363; CHECK-SD-NEXT:    umull2 v4.8h, v0.16b, v1.16b2364; CHECK-SD-NEXT:    umull v0.8h, v0.8b, v1.8b2365; CHECK-SD-NEXT:    umull v1.8h, v3.8b, v2.8b2366; CHECK-SD-NEXT:    umull2 v2.8h, v3.16b, v2.16b2367; CHECK-SD-NEXT:    ushll v3.4s, v4.4h, #02368; CHECK-SD-NEXT:    uaddl2 v4.4s, v1.8h, v0.8h2369; CHECK-SD-NEXT:    uaddl v0.4s, v1.4h, v0.4h2370; CHECK-SD-NEXT:    mov v5.s[0], v3.s[0]2371; CHECK-SD-NEXT:    uaddw2 v1.4s, v4.4s, v2.8h2372; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s2373; CHECK-SD-NEXT:    uaddw v2.4s, v5.4s, v2.4h2374; CHECK-SD-NEXT:    add v0.4s, v0.4s, v2.4s2375; CHECK-SD-NEXT:    addv s0, v0.4s2376; CHECK-SD-NEXT:    fmov w8, s02377; CHECK-SD-NEXT:    add w0, w8, w22378; CHECK-SD-NEXT:    ret2379;2380; CHECK-GI-LABEL: test_udot_v25i8:2381; CHECK-GI:       // %bb.0: // %entry2382; CHECK-GI-NEXT:    sub sp, sp, #1122383; CHECK-GI-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill2384; CHECK-GI-NEXT:    stp x28, x27, [sp, #32] // 16-byte Folded Spill2385; CHECK-GI-NEXT:    stp x26, x25, [sp, #48] // 16-byte Folded Spill2386; CHECK-GI-NEXT:    stp x24, x23, [sp, #64] // 16-byte Folded Spill2387; CHECK-GI-NEXT:    stp x22, x21, [sp, #80] // 16-byte Folded Spill2388; CHECK-GI-NEXT:    stp x20, x19, [sp, #96] // 16-byte Folded Spill2389; CHECK-GI-NEXT:    .cfi_def_cfa_offset 1122390; CHECK-GI-NEXT:    .cfi_offset w19, -82391; CHECK-GI-NEXT:    .cfi_offset w20, -162392; CHECK-GI-NEXT:    .cfi_offset w21, -242393; CHECK-GI-NEXT:    .cfi_offset w22, -322394; CHECK-GI-NEXT:    .cfi_offset w23, -402395; CHECK-GI-NEXT:    .cfi_offset w24, -482396; CHECK-GI-NEXT:    .cfi_offset w25, -562397; CHECK-GI-NEXT:    .cfi_offset w26, -642398; CHECK-GI-NEXT:    .cfi_offset w27, -722399; CHECK-GI-NEXT:    .cfi_offset w28, -802400; CHECK-GI-NEXT:    .cfi_offset w30, -882401; CHECK-GI-NEXT:    .cfi_offset w29, -962402; CHECK-GI-NEXT:    ldp q2, q1, [x1]2403; CHECK-GI-NEXT:    movi d0, #00000000000000002404; CHECK-GI-NEXT:    str w2, [sp, #12] // 4-byte Spill2405; CHECK-GI-NEXT:    mov b6, v2.b[3]2406; CHECK-GI-NEXT:    mov b7, v2.b[4]2407; CHECK-GI-NEXT:    mov b16, v2.b[5]2408; CHECK-GI-NEXT:    mov b19, v2.b[8]2409; CHECK-GI-NEXT:    mov b4, v2.b[1]2410; CHECK-GI-NEXT:    mov b5, v2.b[2]2411; CHECK-GI-NEXT:    mov b17, v2.b[6]2412; CHECK-GI-NEXT:    mov b18, v2.b[7]2413; CHECK-GI-NEXT:    mov b20, v2.b[9]2414; CHECK-GI-NEXT:    mov b21, v2.b[10]2415; CHECK-GI-NEXT:    mov b22, v2.b[11]2416; CHECK-GI-NEXT:    fmov w7, s22417; CHECK-GI-NEXT:    fmov w13, s62418; CHECK-GI-NEXT:    mov b6, v2.b[12]2419; CHECK-GI-NEXT:    fmov w2, s72420; CHECK-GI-NEXT:    mov b7, v2.b[13]2421; CHECK-GI-NEXT:    fmov w11, s162422; CHECK-GI-NEXT:    mov b16, v2.b[14]2423; CHECK-GI-NEXT:    mov b23, v2.b[15]2424; CHECK-GI-NEXT:    ldp q3, q2, [x0]2425; CHECK-GI-NEXT:    fmov w26, s192426; CHECK-GI-NEXT:    fmov w19, s42427; CHECK-GI-NEXT:    stp s17, s18, [sp, #4] // 8-byte Folded Spill2428; CHECK-GI-NEXT:    fmov w29, s52429; CHECK-GI-NEXT:    fmov w24, s202430; CHECK-GI-NEXT:    uxtb w8, w72431; CHECK-GI-NEXT:    mov b4, v3.b[2]2432; CHECK-GI-NEXT:    mov b5, v3.b[1]2433; CHECK-GI-NEXT:    uxtb w13, w132434; CHECK-GI-NEXT:    mov b17, v1.b[1]2435; CHECK-GI-NEXT:    fmov w22, s212436; CHECK-GI-NEXT:    uxtb w26, w262437; CHECK-GI-NEXT:    mov b18, v1.b[2]2438; CHECK-GI-NEXT:    fmov w18, s222439; CHECK-GI-NEXT:    uxtb w24, w242440; CHECK-GI-NEXT:    mov b19, v1.b[3]2441; CHECK-GI-NEXT:    fmov w16, s62442; CHECK-GI-NEXT:    uxtb w19, w192443; CHECK-GI-NEXT:    mov b21, v1.b[4]2444; CHECK-GI-NEXT:    fmov w15, s72445; CHECK-GI-NEXT:    uxtb w22, w222446; CHECK-GI-NEXT:    mov b7, v1.b[5]2447; CHECK-GI-NEXT:    mov b6, v3.b[3]2448; CHECK-GI-NEXT:    uxtb w11, w112449; CHECK-GI-NEXT:    fmov w12, s232450; CHECK-GI-NEXT:    mov b22, v1.b[6]2451; CHECK-GI-NEXT:    mov b23, v1.b[7]2452; CHECK-GI-NEXT:    mov b20, v3.b[4]2453; CHECK-GI-NEXT:    fmov w28, s42454; CHECK-GI-NEXT:    fmov s4, w262455; CHECK-GI-NEXT:    fmov w14, s162456; CHECK-GI-NEXT:    fmov w27, s172457; CHECK-GI-NEXT:    fmov w5, s182458; CHECK-GI-NEXT:    uxtb w12, w122459; CHECK-GI-NEXT:    fmov w4, s192460; CHECK-GI-NEXT:    mov b19, v3.b[5]2461; CHECK-GI-NEXT:    uxtb w28, w282462; CHECK-GI-NEXT:    fmov w3, s212463; CHECK-GI-NEXT:    mov b18, v3.b[6]2464; CHECK-GI-NEXT:    uxtb w27, w272465; CHECK-GI-NEXT:    uxtb w5, w52466; CHECK-GI-NEXT:    fmov w1, s72467; CHECK-GI-NEXT:    mov b16, v3.b[7]2468; CHECK-GI-NEXT:    fmov w0, s222469; CHECK-GI-NEXT:    mov b17, v3.b[8]2470; CHECK-GI-NEXT:    fmov w17, s232471; CHECK-GI-NEXT:    mov b7, v3.b[9]2472; CHECK-GI-NEXT:    fmov w30, s52473; CHECK-GI-NEXT:    mov b5, v3.b[10]2474; CHECK-GI-NEXT:    mov b21, v3.b[11]2475; CHECK-GI-NEXT:    fmov w25, s62476; CHECK-GI-NEXT:    mov b6, v3.b[12]2477; CHECK-GI-NEXT:    fmov w23, s202478; CHECK-GI-NEXT:    mov b20, v3.b[13]2479; CHECK-GI-NEXT:    mov b22, v3.b[14]2480; CHECK-GI-NEXT:    fmov w6, s32481; CHECK-GI-NEXT:    mov b23, v3.b[15]2482; CHECK-GI-NEXT:    fmov s3, w82483; CHECK-GI-NEXT:    fmov w8, s12484; CHECK-GI-NEXT:    mov v4.h[1], w242485; CHECK-GI-NEXT:    fmov w21, s192486; CHECK-GI-NEXT:    mov b19, v2.b[1]2487; CHECK-GI-NEXT:    fmov w9, s172488; CHECK-GI-NEXT:    fmov w24, s62489; CHECK-GI-NEXT:    fmov w7, s162490; CHECK-GI-NEXT:    mov b16, v2.b[2]2491; CHECK-GI-NEXT:    uxtb w8, w82492; CHECK-GI-NEXT:    mov v3.h[1], w192493; CHECK-GI-NEXT:    uxtb w19, w292494; CHECK-GI-NEXT:    uxtb w9, w92495; CHECK-GI-NEXT:    fmov w29, s52496; CHECK-GI-NEXT:    mov v4.h[2], w222497; CHECK-GI-NEXT:    uxtb w22, w62498; CHECK-GI-NEXT:    fmov s5, w82499; CHECK-GI-NEXT:    fmov w10, s72500; CHECK-GI-NEXT:    fmov s7, w92501; CHECK-GI-NEXT:    fmov w9, s162502; CHECK-GI-NEXT:    fmov w20, s182503; CHECK-GI-NEXT:    uxtb w29, w292504; CHECK-GI-NEXT:    fmov s6, w222505; CHECK-GI-NEXT:    fmov w22, s22506; CHECK-GI-NEXT:    uxtb w10, w102507; CHECK-GI-NEXT:    mov v5.h[1], w272508; CHECK-GI-NEXT:    uxtb w27, w302509; CHECK-GI-NEXT:    uxtb w9, w92510; CHECK-GI-NEXT:    mov b18, v2.b[3]2511; CHECK-GI-NEXT:    mov v3.h[2], w192512; CHECK-GI-NEXT:    uxtb w22, w222513; CHECK-GI-NEXT:    mov v6.h[1], w272514; CHECK-GI-NEXT:    fmov w27, s192515; CHECK-GI-NEXT:    mov v7.h[1], w102516; CHECK-GI-NEXT:    fmov w26, s212517; CHECK-GI-NEXT:    mov b17, v2.b[4]2518; CHECK-GI-NEXT:    fmov s16, w222519; CHECK-GI-NEXT:    mov v5.h[2], w52520; CHECK-GI-NEXT:    uxtb w5, w252521; CHECK-GI-NEXT:    uxtb w27, w272522; CHECK-GI-NEXT:    fmov w10, s182523; CHECK-GI-NEXT:    mov v3.h[3], w132524; CHECK-GI-NEXT:    uxtb w13, w42525; CHECK-GI-NEXT:    mov v6.h[2], w282526; CHECK-GI-NEXT:    fmov w8, s202527; CHECK-GI-NEXT:    mov v16.h[1], w272528; CHECK-GI-NEXT:    mov v7.h[2], w292529; CHECK-GI-NEXT:    mov b20, v2.b[5]2530; CHECK-GI-NEXT:    uxtb w10, w102531; CHECK-GI-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload2532; CHECK-GI-NEXT:    uxtb w8, w82533; CHECK-GI-NEXT:    fmov w22, s172534; CHECK-GI-NEXT:    mov v5.h[3], w132535; CHECK-GI-NEXT:    uxtb w13, w22536; CHECK-GI-NEXT:    mov v6.h[3], w52537; CHECK-GI-NEXT:    mov b21, v2.b[6]2538; CHECK-GI-NEXT:    mov v16.h[2], w92539; CHECK-GI-NEXT:    uxtb w9, w182540; CHECK-GI-NEXT:    uxtb w18, w232541; CHECK-GI-NEXT:    mov v3.h[4], w132542; CHECK-GI-NEXT:    uxtb w13, w242543; CHECK-GI-NEXT:    fmov w27, s202544; CHECK-GI-NEXT:    ldp x24, x23, [sp, #64] // 16-byte Folded Reload2545; CHECK-GI-NEXT:    mov v4.h[3], w92546; CHECK-GI-NEXT:    uxtb w9, w262547; CHECK-GI-NEXT:    ldp x26, x25, [sp, #48] // 16-byte Folded Reload2548; CHECK-GI-NEXT:    mov v16.h[3], w102549; CHECK-GI-NEXT:    uxtb w10, w32550; CHECK-GI-NEXT:    mov v6.h[4], w182551; CHECK-GI-NEXT:    ldr w18, [sp, #4] // 4-byte Reload2552; CHECK-GI-NEXT:    mov v7.h[3], w92553; CHECK-GI-NEXT:    uxtb w9, w162554; CHECK-GI-NEXT:    uxtb w16, w222555; CHECK-GI-NEXT:    mov v5.h[4], w102556; CHECK-GI-NEXT:    uxtb w10, w152557; CHECK-GI-NEXT:    uxtb w18, w182558; CHECK-GI-NEXT:    mov v4.h[4], w92559; CHECK-GI-NEXT:    uxtb w9, w212560; CHECK-GI-NEXT:    ldp x22, x21, [sp, #80] // 16-byte Folded Reload2561; CHECK-GI-NEXT:    mov v16.h[4], w162562; CHECK-GI-NEXT:    mov v7.h[4], w132563; CHECK-GI-NEXT:    ldr w13, [sp, #8] // 4-byte Reload2564; CHECK-GI-NEXT:    mov v6.h[5], w92565; CHECK-GI-NEXT:    uxtb w9, w12566; CHECK-GI-NEXT:    mov v3.h[5], w112567; CHECK-GI-NEXT:    uxtb w11, w272568; CHECK-GI-NEXT:    fmov w19, s222569; CHECK-GI-NEXT:    fmov w28, s212570; CHECK-GI-NEXT:    uxtb w13, w132571; CHECK-GI-NEXT:    mov b17, v2.b[7]2572; CHECK-GI-NEXT:    mov v5.h[5], w92573; CHECK-GI-NEXT:    uxtb w9, w02574; CHECK-GI-NEXT:    mov v4.h[5], w102575; CHECK-GI-NEXT:    uxtb w10, w202576; CHECK-GI-NEXT:    mov v7.h[5], w82577; CHECK-GI-NEXT:    mov v16.h[5], w112578; CHECK-GI-NEXT:    uxtb w8, w142579; CHECK-GI-NEXT:    uxtb w11, w282580; CHECK-GI-NEXT:    mov v6.h[6], w102581; CHECK-GI-NEXT:    uxtb w10, w192582; CHECK-GI-NEXT:    fmov w6, s232583; CHECK-GI-NEXT:    mov v5.h[6], w92584; CHECK-GI-NEXT:    fmov w9, s172585; CHECK-GI-NEXT:    mov v3.h[6], w182586; CHECK-GI-NEXT:    mov v4.h[6], w82587; CHECK-GI-NEXT:    uxtb w8, w72588; CHECK-GI-NEXT:    mov v7.h[6], w102589; CHECK-GI-NEXT:    mov v16.h[6], w112590; CHECK-GI-NEXT:    uxtb w10, w62591; CHECK-GI-NEXT:    mov v0.s[1], wzr2592; CHECK-GI-NEXT:    mov v6.h[7], w82593; CHECK-GI-NEXT:    uxtb w8, w172594; CHECK-GI-NEXT:    uxtb w9, w92595; CHECK-GI-NEXT:    mov v3.h[7], w132596; CHECK-GI-NEXT:    ldp x28, x27, [sp, #32] // 16-byte Folded Reload2597; CHECK-GI-NEXT:    mov v4.h[7], w122598; CHECK-GI-NEXT:    mov v5.h[7], w82599; CHECK-GI-NEXT:    mov v7.h[7], w102600; CHECK-GI-NEXT:    mov v16.h[7], w92601; CHECK-GI-NEXT:    umov w8, v1.b[8]2602; CHECK-GI-NEXT:    umov w9, v2.b[8]2603; CHECK-GI-NEXT:    mov v0.s[2], wzr2604; CHECK-GI-NEXT:    mul v3.8h, v3.8h, v6.8h2605; CHECK-GI-NEXT:    mul v2.8h, v4.8h, v7.8h2606; CHECK-GI-NEXT:    mul v1.8h, v5.8h, v16.8h2607; CHECK-GI-NEXT:    mul w15, w8, w92608; CHECK-GI-NEXT:    mov v0.s[3], wzr2609; CHECK-GI-NEXT:    umov w16, v3.h[0]2610; CHECK-GI-NEXT:    umov w18, v3.h[4]2611; CHECK-GI-NEXT:    umov w17, v3.h[1]2612; CHECK-GI-NEXT:    umov w1, v2.h[0]2613; CHECK-GI-NEXT:    umov w3, v2.h[4]2614; CHECK-GI-NEXT:    umov w0, v3.h[5]2615; CHECK-GI-NEXT:    umov w5, v1.h[0]2616; CHECK-GI-NEXT:    umov w7, v1.h[4]2617; CHECK-GI-NEXT:    umov w2, v2.h[1]2618; CHECK-GI-NEXT:    umov w4, v2.h[5]2619; CHECK-GI-NEXT:    umov w6, v1.h[1]2620; CHECK-GI-NEXT:    umov w19, v1.h[5]2621; CHECK-GI-NEXT:    umov w10, v3.h[2]2622; CHECK-GI-NEXT:    umov w8, v3.h[3]2623; CHECK-GI-NEXT:    umov w11, v3.h[6]2624; CHECK-GI-NEXT:    umov w9, v3.h[7]2625; CHECK-GI-NEXT:    fmov s3, w162626; CHECK-GI-NEXT:    fmov s4, w182627; CHECK-GI-NEXT:    fmov s5, w12628; CHECK-GI-NEXT:    fmov s6, w32629; CHECK-GI-NEXT:    fmov s7, w52630; CHECK-GI-NEXT:    fmov s16, w72631; CHECK-GI-NEXT:    fmov s17, w152632; CHECK-GI-NEXT:    umov w12, v2.h[2]2633; CHECK-GI-NEXT:    umov w13, v2.h[6]2634; CHECK-GI-NEXT:    umov w14, v1.h[2]2635; CHECK-GI-NEXT:    umov w16, v1.h[6]2636; CHECK-GI-NEXT:    mov v3.s[1], w172637; CHECK-GI-NEXT:    mov v4.s[1], w02638; CHECK-GI-NEXT:    mov v5.s[1], w22639; CHECK-GI-NEXT:    mov v6.s[1], w42640; CHECK-GI-NEXT:    mov v7.s[1], w62641; CHECK-GI-NEXT:    mov v16.s[1], w192642; CHECK-GI-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload2643; CHECK-GI-NEXT:    mov v17.s[1], wzr2644; CHECK-GI-NEXT:    umov w15, v2.h[3]2645; CHECK-GI-NEXT:    umov w17, v2.h[7]2646; CHECK-GI-NEXT:    umov w18, v1.h[3]2647; CHECK-GI-NEXT:    umov w0, v1.h[7]2648; CHECK-GI-NEXT:    mov v3.s[2], w102649; CHECK-GI-NEXT:    mov v4.s[2], w112650; CHECK-GI-NEXT:    mov v5.s[2], w122651; CHECK-GI-NEXT:    mov v6.s[2], w132652; CHECK-GI-NEXT:    mov v7.s[2], w142653; CHECK-GI-NEXT:    mov v16.s[2], w162654; CHECK-GI-NEXT:    mov v17.s[2], wzr2655; CHECK-GI-NEXT:    mov v3.s[3], w82656; CHECK-GI-NEXT:    mov v4.s[3], w92657; CHECK-GI-NEXT:    ldr w9, [sp, #12] // 4-byte Reload2658; CHECK-GI-NEXT:    mov v5.s[3], w152659; CHECK-GI-NEXT:    mov v6.s[3], w172660; CHECK-GI-NEXT:    mov v7.s[3], w182661; CHECK-GI-NEXT:    mov v16.s[3], w02662; CHECK-GI-NEXT:    mov v17.s[3], wzr2663; CHECK-GI-NEXT:    add v1.4s, v3.4s, v4.4s2664; CHECK-GI-NEXT:    add v2.4s, v5.4s, v6.4s2665; CHECK-GI-NEXT:    add v3.4s, v7.4s, v16.4s2666; CHECK-GI-NEXT:    add v0.4s, v17.4s, v0.4s2667; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s2668; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s2669; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s2670; CHECK-GI-NEXT:    addv s0, v0.4s2671; CHECK-GI-NEXT:    fmov w8, s02672; CHECK-GI-NEXT:    add w0, w8, w92673; CHECK-GI-NEXT:    add sp, sp, #1122674; CHECK-GI-NEXT:    ret2675entry:2676  %0 = load <25 x i8>, ptr %a2677  %1 = zext <25 x i8> %0 to <25 x i32>2678  %2 = load <25 x i8>, ptr %b2679  %3 = zext <25 x i8> %2 to <25 x i32>2680  %4 = mul nuw nsw <25 x i32> %3, %12681  %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)2682  %op.extra = add i32 %5, %sum2683  ret i32 %op.extra2684}2685 2686define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) {2687; CHECK-SD-LABEL: test_udot_v25i8_nomla:2688; CHECK-SD:       // %bb.0: // %entry2689; CHECK-SD-NEXT:    ldp q2, q1, [x0]2690; CHECK-SD-NEXT:    movi v0.2d, #00000000000000002691; CHECK-SD-NEXT:    ushll2 v3.8h, v1.16b, #02692; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #02693; CHECK-SD-NEXT:    ushll v4.8h, v2.8b, #02694; CHECK-SD-NEXT:    ushll2 v2.8h, v2.16b, #02695; CHECK-SD-NEXT:    ushll v3.4s, v3.4h, #02696; CHECK-SD-NEXT:    uaddl2 v5.4s, v4.8h, v1.8h2697; CHECK-SD-NEXT:    uaddl v1.4s, v4.4h, v1.4h2698; CHECK-SD-NEXT:    mov v0.s[0], v3.s[0]2699; CHECK-SD-NEXT:    uaddw2 v3.4s, v5.4s, v2.8h2700; CHECK-SD-NEXT:    add v1.4s, v1.4s, v3.4s2701; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v2.4h2702; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s2703; CHECK-SD-NEXT:    addv s0, v0.4s2704; CHECK-SD-NEXT:    fmov w0, s02705; CHECK-SD-NEXT:    ret2706;2707; CHECK-GI-LABEL: test_udot_v25i8_nomla:2708; CHECK-GI:       // %bb.0: // %entry2709; CHECK-GI-NEXT:    str x19, [sp, #-16]! // 8-byte Folded Spill2710; CHECK-GI-NEXT:    .cfi_def_cfa_offset 162711; CHECK-GI-NEXT:    .cfi_offset w19, -162712; CHECK-GI-NEXT:    ldp q2, q1, [x0]2713; CHECK-GI-NEXT:    movi d0, #00000000000000002714; CHECK-GI-NEXT:    umov w15, v2.b[0]2715; CHECK-GI-NEXT:    umov w17, v2.b[4]2716; CHECK-GI-NEXT:    umov w0, v2.b[8]2717; CHECK-GI-NEXT:    umov w2, v2.b[12]2718; CHECK-GI-NEXT:    umov w4, v1.b[0]2719; CHECK-GI-NEXT:    umov w6, v1.b[4]2720; CHECK-GI-NEXT:    umov w19, v1.b[8]2721; CHECK-GI-NEXT:    umov w16, v2.b[1]2722; CHECK-GI-NEXT:    umov w18, v2.b[5]2723; CHECK-GI-NEXT:    umov w1, v2.b[9]2724; CHECK-GI-NEXT:    umov w3, v2.b[13]2725; CHECK-GI-NEXT:    umov w5, v1.b[1]2726; CHECK-GI-NEXT:    umov w7, v1.b[5]2727; CHECK-GI-NEXT:    fmov s3, w152728; CHECK-GI-NEXT:    fmov s4, w172729; CHECK-GI-NEXT:    fmov s5, w02730; CHECK-GI-NEXT:    fmov s6, w22731; CHECK-GI-NEXT:    fmov s7, w42732; CHECK-GI-NEXT:    fmov s16, w62733; CHECK-GI-NEXT:    fmov s17, w192734; CHECK-GI-NEXT:    umov w10, v2.b[2]2735; CHECK-GI-NEXT:    umov w11, v2.b[6]2736; CHECK-GI-NEXT:    umov w12, v2.b[10]2737; CHECK-GI-NEXT:    umov w13, v2.b[14]2738; CHECK-GI-NEXT:    umov w14, v1.b[2]2739; CHECK-GI-NEXT:    umov w15, v1.b[6]2740; CHECK-GI-NEXT:    mov v3.s[1], w162741; CHECK-GI-NEXT:    mov v4.s[1], w182742; CHECK-GI-NEXT:    mov v5.s[1], w12743; CHECK-GI-NEXT:    mov v6.s[1], w32744; CHECK-GI-NEXT:    mov v7.s[1], w52745; CHECK-GI-NEXT:    mov v16.s[1], w72746; CHECK-GI-NEXT:    mov v17.s[1], wzr2747; CHECK-GI-NEXT:    mov v0.s[1], wzr2748; CHECK-GI-NEXT:    umov w8, v2.b[3]2749; CHECK-GI-NEXT:    umov w9, v2.b[7]2750; CHECK-GI-NEXT:    umov w16, v2.b[11]2751; CHECK-GI-NEXT:    umov w17, v2.b[15]2752; CHECK-GI-NEXT:    umov w18, v1.b[3]2753; CHECK-GI-NEXT:    umov w0, v1.b[7]2754; CHECK-GI-NEXT:    mov v3.s[2], w102755; CHECK-GI-NEXT:    mov v4.s[2], w112756; CHECK-GI-NEXT:    mov v5.s[2], w122757; CHECK-GI-NEXT:    mov v6.s[2], w132758; CHECK-GI-NEXT:    mov v7.s[2], w142759; CHECK-GI-NEXT:    mov v16.s[2], w152760; CHECK-GI-NEXT:    mov v17.s[2], wzr2761; CHECK-GI-NEXT:    mov v0.s[2], wzr2762; CHECK-GI-NEXT:    mov v3.s[3], w82763; CHECK-GI-NEXT:    mov v4.s[3], w92764; CHECK-GI-NEXT:    mov v5.s[3], w162765; CHECK-GI-NEXT:    mov v6.s[3], w172766; CHECK-GI-NEXT:    mov v7.s[3], w182767; CHECK-GI-NEXT:    mov v16.s[3], w02768; CHECK-GI-NEXT:    mov v17.s[3], wzr2769; CHECK-GI-NEXT:    mov v0.s[3], wzr2770; CHECK-GI-NEXT:    add v1.4s, v3.4s, v4.4s2771; CHECK-GI-NEXT:    add v2.4s, v5.4s, v6.4s2772; CHECK-GI-NEXT:    add v3.4s, v7.4s, v16.4s2773; CHECK-GI-NEXT:    add v0.4s, v17.4s, v0.4s2774; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s2775; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s2776; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s2777; CHECK-GI-NEXT:    addv s0, v0.4s2778; CHECK-GI-NEXT:    fmov w0, s02779; CHECK-GI-NEXT:    ldr x19, [sp], #16 // 8-byte Folded Reload2780; CHECK-GI-NEXT:    ret2781entry:2782  %0 = load <25 x i8>, ptr %a12783  %1 = zext <25 x i8> %0 to <25 x i32>2784  %2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %1)2785  ret i32 %22786}2787define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {2788; CHECK-SD-LABEL: test_sdot_v25i8:2789; CHECK-SD:       // %bb.0: // %entry2790; CHECK-SD-NEXT:    ldp q3, q0, [x1]2791; CHECK-SD-NEXT:    movi v5.2d, #00000000000000002792; CHECK-SD-NEXT:    ldp q2, q1, [x0]2793; CHECK-SD-NEXT:    smull2 v4.8h, v0.16b, v1.16b2794; CHECK-SD-NEXT:    smull v0.8h, v0.8b, v1.8b2795; CHECK-SD-NEXT:    smull v1.8h, v3.8b, v2.8b2796; CHECK-SD-NEXT:    smull2 v2.8h, v3.16b, v2.16b2797; CHECK-SD-NEXT:    sshll v3.4s, v4.4h, #02798; CHECK-SD-NEXT:    saddl2 v4.4s, v1.8h, v0.8h2799; CHECK-SD-NEXT:    saddl v0.4s, v1.4h, v0.4h2800; CHECK-SD-NEXT:    mov v5.s[0], v3.s[0]2801; CHECK-SD-NEXT:    saddw2 v1.4s, v4.4s, v2.8h2802; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s2803; CHECK-SD-NEXT:    saddw v2.4s, v5.4s, v2.4h2804; CHECK-SD-NEXT:    add v0.4s, v0.4s, v2.4s2805; CHECK-SD-NEXT:    addv s0, v0.4s2806; CHECK-SD-NEXT:    fmov w8, s02807; CHECK-SD-NEXT:    add w0, w8, w22808; CHECK-SD-NEXT:    ret2809;2810; CHECK-GI-LABEL: test_sdot_v25i8:2811; CHECK-GI:       // %bb.0: // %entry2812; CHECK-GI-NEXT:    sub sp, sp, #1122813; CHECK-GI-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill2814; CHECK-GI-NEXT:    stp x28, x27, [sp, #32] // 16-byte Folded Spill2815; CHECK-GI-NEXT:    stp x26, x25, [sp, #48] // 16-byte Folded Spill2816; CHECK-GI-NEXT:    stp x24, x23, [sp, #64] // 16-byte Folded Spill2817; CHECK-GI-NEXT:    stp x22, x21, [sp, #80] // 16-byte Folded Spill2818; CHECK-GI-NEXT:    stp x20, x19, [sp, #96] // 16-byte Folded Spill2819; CHECK-GI-NEXT:    .cfi_def_cfa_offset 1122820; CHECK-GI-NEXT:    .cfi_offset w19, -82821; CHECK-GI-NEXT:    .cfi_offset w20, -162822; CHECK-GI-NEXT:    .cfi_offset w21, -242823; CHECK-GI-NEXT:    .cfi_offset w22, -322824; CHECK-GI-NEXT:    .cfi_offset w23, -402825; CHECK-GI-NEXT:    .cfi_offset w24, -482826; CHECK-GI-NEXT:    .cfi_offset w25, -562827; CHECK-GI-NEXT:    .cfi_offset w26, -642828; CHECK-GI-NEXT:    .cfi_offset w27, -722829; CHECK-GI-NEXT:    .cfi_offset w28, -802830; CHECK-GI-NEXT:    .cfi_offset w30, -882831; CHECK-GI-NEXT:    .cfi_offset w29, -962832; CHECK-GI-NEXT:    ldp q2, q1, [x1]2833; CHECK-GI-NEXT:    movi d0, #00000000000000002834; CHECK-GI-NEXT:    str w2, [sp, #12] // 4-byte Spill2835; CHECK-GI-NEXT:    mov b5, v2.b[2]2836; CHECK-GI-NEXT:    mov b6, v2.b[3]2837; CHECK-GI-NEXT:    mov b7, v2.b[4]2838; CHECK-GI-NEXT:    mov b16, v2.b[5]2839; CHECK-GI-NEXT:    mov b17, v2.b[6]2840; CHECK-GI-NEXT:    mov b18, v2.b[7]2841; CHECK-GI-NEXT:    mov b19, v2.b[8]2842; CHECK-GI-NEXT:    mov b20, v2.b[9]2843; CHECK-GI-NEXT:    mov b21, v2.b[15]2844; CHECK-GI-NEXT:    mov b3, v2.b[1]2845; CHECK-GI-NEXT:    fmov w19, s22846; CHECK-GI-NEXT:    mov b22, v1.b[6]2847; CHECK-GI-NEXT:    fmov w6, s52848; CHECK-GI-NEXT:    mov b5, v2.b[10]2849; CHECK-GI-NEXT:    fmov w14, s62850; CHECK-GI-NEXT:    mov b6, v2.b[11]2851; CHECK-GI-NEXT:    fmov w2, s72852; CHECK-GI-NEXT:    stp s17, s18, [sp, #4] // 8-byte Folded Spill2853; CHECK-GI-NEXT:    mov b7, v2.b[12]2854; CHECK-GI-NEXT:    fmov w11, s162855; CHECK-GI-NEXT:    sxtb w28, w192856; CHECK-GI-NEXT:    mov b16, v2.b[13]2857; CHECK-GI-NEXT:    mov b18, v1.b[1]2858; CHECK-GI-NEXT:    sxtb w6, w62859; CHECK-GI-NEXT:    mov b17, v2.b[14]2860; CHECK-GI-NEXT:    ldp q4, q2, [x0]2861; CHECK-GI-NEXT:    fmov w25, s192862; CHECK-GI-NEXT:    fmov w24, s202863; CHECK-GI-NEXT:    fmov w22, s52864; CHECK-GI-NEXT:    mov b5, v1.b[2]2865; CHECK-GI-NEXT:    fmov w0, s62866; CHECK-GI-NEXT:    sxtb w14, w142867; CHECK-GI-NEXT:    mov b20, v1.b[3]2868; CHECK-GI-NEXT:    fmov w16, s72869; CHECK-GI-NEXT:    mov b7, v1.b[4]2870; CHECK-GI-NEXT:    fmov w15, s162871; CHECK-GI-NEXT:    sxtb w25, w252872; CHECK-GI-NEXT:    sxtb w24, w242873; CHECK-GI-NEXT:    mov b16, v1.b[5]2874; CHECK-GI-NEXT:    fmov w13, s212875; CHECK-GI-NEXT:    sxtb w22, w222876; CHECK-GI-NEXT:    mov b6, v4.b[2]2877; CHECK-GI-NEXT:    fmov w26, s182878; CHECK-GI-NEXT:    sxtb w0, w02879; CHECK-GI-NEXT:    mov b21, v1.b[7]2880; CHECK-GI-NEXT:    mov b18, v4.b[4]2881; CHECK-GI-NEXT:    fmov w7, s32882; CHECK-GI-NEXT:    mov b3, v4.b[1]2883; CHECK-GI-NEXT:    fmov w12, s172884; CHECK-GI-NEXT:    fmov w5, s52885; CHECK-GI-NEXT:    mov b19, v4.b[3]2886; CHECK-GI-NEXT:    fmov w4, s202887; CHECK-GI-NEXT:    fmov w3, s72888; CHECK-GI-NEXT:    sxtb w29, w72889; CHECK-GI-NEXT:    mov b17, v4.b[5]2890; CHECK-GI-NEXT:    fmov w1, s162891; CHECK-GI-NEXT:    sxtb w5, w52892; CHECK-GI-NEXT:    mov b16, v4.b[6]2893; CHECK-GI-NEXT:    fmov w18, s222894; CHECK-GI-NEXT:    mov b7, v4.b[7]2895; CHECK-GI-NEXT:    fmov w17, s212896; CHECK-GI-NEXT:    mov b5, v4.b[8]2897; CHECK-GI-NEXT:    mov b20, v4.b[9]2898; CHECK-GI-NEXT:    fmov w27, s62899; CHECK-GI-NEXT:    mov b6, v4.b[10]2900; CHECK-GI-NEXT:    mov b21, v4.b[11]2901; CHECK-GI-NEXT:    fmov w21, s182902; CHECK-GI-NEXT:    mov b18, v4.b[12]2903; CHECK-GI-NEXT:    mov b22, v4.b[13]2904; CHECK-GI-NEXT:    mov b23, v4.b[14]2905; CHECK-GI-NEXT:    fmov w10, s42906; CHECK-GI-NEXT:    sxtb w27, w272907; CHECK-GI-NEXT:    mov b24, v4.b[15]2908; CHECK-GI-NEXT:    fmov s4, w252909; CHECK-GI-NEXT:    fmov w30, s32910; CHECK-GI-NEXT:    fmov s3, w282911; CHECK-GI-NEXT:    fmov w9, s52912; CHECK-GI-NEXT:    sxtb w10, w102913; CHECK-GI-NEXT:    fmov w7, s72914; CHECK-GI-NEXT:    mov b7, v2.b[1]2915; CHECK-GI-NEXT:    mov v4.h[1], w242916; CHECK-GI-NEXT:    fmov w24, s12917; CHECK-GI-NEXT:    fmov w8, s202918; CHECK-GI-NEXT:    sxtb w9, w92919; CHECK-GI-NEXT:    mov v3.h[1], w292920; CHECK-GI-NEXT:    fmov w29, s62921; CHECK-GI-NEXT:    fmov s6, w102922; CHECK-GI-NEXT:    fmov w10, s22923; CHECK-GI-NEXT:    fmov w19, s162924; CHECK-GI-NEXT:    sxtb w24, w242925; CHECK-GI-NEXT:    sxtb w8, w82926; CHECK-GI-NEXT:    mov b16, v2.b[3]2927; CHECK-GI-NEXT:    sxtb w29, w292928; CHECK-GI-NEXT:    fmov w23, s192929; CHECK-GI-NEXT:    mov b19, v2.b[2]2930; CHECK-GI-NEXT:    sxtb w10, w102931; CHECK-GI-NEXT:    fmov s5, w242932; CHECK-GI-NEXT:    sxtb w24, w302933; CHECK-GI-NEXT:    mov v3.h[2], w62934; CHECK-GI-NEXT:    sxtb w6, w262935; CHECK-GI-NEXT:    fmov w28, s212936; CHECK-GI-NEXT:    sxtb w23, w232937; CHECK-GI-NEXT:    mov v6.h[1], w242938; CHECK-GI-NEXT:    fmov w24, s72939; CHECK-GI-NEXT:    fmov s7, w92940; CHECK-GI-NEXT:    fmov w9, s192941; CHECK-GI-NEXT:    mov v5.h[1], w62942; CHECK-GI-NEXT:    mov v4.h[2], w222943; CHECK-GI-NEXT:    fmov w20, s172944; CHECK-GI-NEXT:    mov b17, v2.b[4]2945; CHECK-GI-NEXT:    sxtb w24, w242946; CHECK-GI-NEXT:    mov v3.h[3], w142947; CHECK-GI-NEXT:    sxtb w14, w22948; CHECK-GI-NEXT:    sxtb w9, w92949; CHECK-GI-NEXT:    mov v7.h[1], w82950; CHECK-GI-NEXT:    fmov w8, s162951; CHECK-GI-NEXT:    fmov s16, w102952; CHECK-GI-NEXT:    mov v6.h[2], w272953; CHECK-GI-NEXT:    mov v5.h[2], w52954; CHECK-GI-NEXT:    fmov w25, s182955; CHECK-GI-NEXT:    mov v4.h[3], w02956; CHECK-GI-NEXT:    sxtb w0, w42957; CHECK-GI-NEXT:    sxtb w8, w82958; CHECK-GI-NEXT:    mov b18, v2.b[5]2959; CHECK-GI-NEXT:    fmov w10, s172960; CHECK-GI-NEXT:    mov v16.h[1], w242961; CHECK-GI-NEXT:    mov v7.h[2], w292962; CHECK-GI-NEXT:    mov v3.h[4], w142963; CHECK-GI-NEXT:    sxtb w14, w252964; CHECK-GI-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload2965; CHECK-GI-NEXT:    sxtb w10, w102966; CHECK-GI-NEXT:    mov v6.h[3], w232967; CHECK-GI-NEXT:    mov v5.h[3], w02968; CHECK-GI-NEXT:    fmov w26, s222969; CHECK-GI-NEXT:    mov b19, v2.b[6]2970; CHECK-GI-NEXT:    fmov w27, s182971; CHECK-GI-NEXT:    mov v16.h[2], w92972; CHECK-GI-NEXT:    sxtb w9, w282973; CHECK-GI-NEXT:    fmov w22, s232974; CHECK-GI-NEXT:    mov b17, v2.b[7]2975; CHECK-GI-NEXT:    fmov w6, s242976; CHECK-GI-NEXT:    mov v0.s[1], wzr2977; CHECK-GI-NEXT:    mov v7.h[3], w92978; CHECK-GI-NEXT:    sxtb w9, w112979; CHECK-GI-NEXT:    sxtb w11, w212980; CHECK-GI-NEXT:    fmov w24, s192981; CHECK-GI-NEXT:    mov v16.h[3], w82982; CHECK-GI-NEXT:    sxtb w8, w162983; CHECK-GI-NEXT:    sxtb w16, w32984; CHECK-GI-NEXT:    mov v6.h[4], w112985; CHECK-GI-NEXT:    ldr w11, [sp, #4] // 4-byte Reload2986; CHECK-GI-NEXT:    mov v3.h[5], w92987; CHECK-GI-NEXT:    sxtb w9, w152988; CHECK-GI-NEXT:    sxtb w15, w272989; CHECK-GI-NEXT:    mov v7.h[4], w142990; CHECK-GI-NEXT:    sxtb w14, w12991; CHECK-GI-NEXT:    sxtb w11, w112992; CHECK-GI-NEXT:    mov v4.h[4], w82993; CHECK-GI-NEXT:    sxtb w8, w202994; CHECK-GI-NEXT:    ldp x28, x27, [sp, #32] // 16-byte Folded Reload2995; CHECK-GI-NEXT:    mov v5.h[4], w162996; CHECK-GI-NEXT:    mov v16.h[4], w102997; CHECK-GI-NEXT:    sxtb w10, w262998; CHECK-GI-NEXT:    ldp x26, x25, [sp, #48] // 16-byte Folded Reload2999; CHECK-GI-NEXT:    mov v6.h[5], w83000; CHECK-GI-NEXT:    ldr w8, [sp, #8] // 4-byte Reload3001; CHECK-GI-NEXT:    mov v7.h[5], w103002; CHECK-GI-NEXT:    sxtb w10, w123003; CHECK-GI-NEXT:    sxtb w12, w183004; CHECK-GI-NEXT:    mov v4.h[5], w93005; CHECK-GI-NEXT:    sxtb w9, w193006; CHECK-GI-NEXT:    mov v5.h[5], w143007; CHECK-GI-NEXT:    sxtb w8, w83008; CHECK-GI-NEXT:    mov v16.h[5], w153009; CHECK-GI-NEXT:    mov v3.h[6], w113010; CHECK-GI-NEXT:    sxtb w11, w223011; CHECK-GI-NEXT:    mov v6.h[6], w93012; CHECK-GI-NEXT:    sxtb w9, w133013; CHECK-GI-NEXT:    sxtb w13, w243014; CHECK-GI-NEXT:    mov v0.s[2], wzr3015; CHECK-GI-NEXT:    mov v7.h[6], w113016; CHECK-GI-NEXT:    fmov w11, s173017; CHECK-GI-NEXT:    mov v4.h[6], w103018; CHECK-GI-NEXT:    sxtb w10, w73019; CHECK-GI-NEXT:    mov v5.h[6], w123020; CHECK-GI-NEXT:    mov v16.h[6], w133021; CHECK-GI-NEXT:    mov v3.h[7], w83022; CHECK-GI-NEXT:    sxtb w8, w63023; CHECK-GI-NEXT:    smov w12, v1.b[8]3024; CHECK-GI-NEXT:    mov v6.h[7], w103025; CHECK-GI-NEXT:    sxtb w10, w173026; CHECK-GI-NEXT:    sxtb w11, w113027; CHECK-GI-NEXT:    mov v4.h[7], w93028; CHECK-GI-NEXT:    mov v7.h[7], w83029; CHECK-GI-NEXT:    smov w8, v2.b[8]3030; CHECK-GI-NEXT:    mov v5.h[7], w103031; CHECK-GI-NEXT:    ldp x22, x21, [sp, #80] // 16-byte Folded Reload3032; CHECK-GI-NEXT:    mov v16.h[7], w113033; CHECK-GI-NEXT:    mov v0.s[3], wzr3034; CHECK-GI-NEXT:    mul v3.8h, v3.8h, v6.8h3035; CHECK-GI-NEXT:    ldp x24, x23, [sp, #64] // 16-byte Folded Reload3036; CHECK-GI-NEXT:    mul v2.8h, v4.8h, v7.8h3037; CHECK-GI-NEXT:    mul w16, w12, w83038; CHECK-GI-NEXT:    mul v1.8h, v5.8h, v16.8h3039; CHECK-GI-NEXT:    smov w17, v3.h[0]3040; CHECK-GI-NEXT:    smov w0, v3.h[4]3041; CHECK-GI-NEXT:    sxth w16, w163042; CHECK-GI-NEXT:    smov w2, v2.h[0]3043; CHECK-GI-NEXT:    smov w4, v2.h[4]3044; CHECK-GI-NEXT:    smov w18, v3.h[1]3045; CHECK-GI-NEXT:    smov w1, v3.h[5]3046; CHECK-GI-NEXT:    smov w3, v2.h[1]3047; CHECK-GI-NEXT:    smov w5, v2.h[5]3048; CHECK-GI-NEXT:    smov w6, v1.h[0]3049; CHECK-GI-NEXT:    smov w19, v1.h[4]3050; CHECK-GI-NEXT:    smov w7, v1.h[1]3051; CHECK-GI-NEXT:    smov w20, v1.h[5]3052; CHECK-GI-NEXT:    smov w10, v3.h[2]3053; CHECK-GI-NEXT:    smov w8, v3.h[3]3054; CHECK-GI-NEXT:    smov w11, v3.h[6]3055; CHECK-GI-NEXT:    smov w9, v3.h[7]3056; CHECK-GI-NEXT:    fmov s3, w173057; CHECK-GI-NEXT:    fmov s4, w03058; CHECK-GI-NEXT:    fmov s5, w23059; CHECK-GI-NEXT:    fmov s6, w43060; CHECK-GI-NEXT:    fmov s7, w63061; CHECK-GI-NEXT:    fmov s16, w193062; CHECK-GI-NEXT:    fmov s17, w163063; CHECK-GI-NEXT:    smov w12, v2.h[2]3064; CHECK-GI-NEXT:    smov w13, v2.h[6]3065; CHECK-GI-NEXT:    smov w14, v1.h[2]3066; CHECK-GI-NEXT:    smov w15, v1.h[6]3067; CHECK-GI-NEXT:    mov v3.s[1], w183068; CHECK-GI-NEXT:    mov v4.s[1], w13069; CHECK-GI-NEXT:    mov v5.s[1], w33070; CHECK-GI-NEXT:    mov v6.s[1], w53071; CHECK-GI-NEXT:    mov v7.s[1], w73072; CHECK-GI-NEXT:    mov v16.s[1], w203073; CHECK-GI-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload3074; CHECK-GI-NEXT:    mov v17.s[1], wzr3075; CHECK-GI-NEXT:    smov w16, v2.h[3]3076; CHECK-GI-NEXT:    smov w17, v2.h[7]3077; CHECK-GI-NEXT:    smov w18, v1.h[3]3078; CHECK-GI-NEXT:    smov w0, v1.h[7]3079; CHECK-GI-NEXT:    mov v3.s[2], w103080; CHECK-GI-NEXT:    mov v4.s[2], w113081; CHECK-GI-NEXT:    mov v5.s[2], w123082; CHECK-GI-NEXT:    mov v6.s[2], w133083; CHECK-GI-NEXT:    mov v7.s[2], w143084; CHECK-GI-NEXT:    mov v16.s[2], w153085; CHECK-GI-NEXT:    mov v17.s[2], wzr3086; CHECK-GI-NEXT:    mov v3.s[3], w83087; CHECK-GI-NEXT:    mov v4.s[3], w93088; CHECK-GI-NEXT:    ldr w9, [sp, #12] // 4-byte Reload3089; CHECK-GI-NEXT:    mov v5.s[3], w163090; CHECK-GI-NEXT:    mov v6.s[3], w173091; CHECK-GI-NEXT:    mov v7.s[3], w183092; CHECK-GI-NEXT:    mov v16.s[3], w03093; CHECK-GI-NEXT:    mov v17.s[3], wzr3094; CHECK-GI-NEXT:    add v1.4s, v3.4s, v4.4s3095; CHECK-GI-NEXT:    add v2.4s, v5.4s, v6.4s3096; CHECK-GI-NEXT:    add v3.4s, v7.4s, v16.4s3097; CHECK-GI-NEXT:    add v0.4s, v17.4s, v0.4s3098; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s3099; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s3100; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s3101; CHECK-GI-NEXT:    addv s0, v0.4s3102; CHECK-GI-NEXT:    fmov w8, s03103; CHECK-GI-NEXT:    add w0, w8, w93104; CHECK-GI-NEXT:    add sp, sp, #1123105; CHECK-GI-NEXT:    ret3106entry:3107  %0 = load <25 x i8>, ptr %a3108  %1 = sext <25 x i8> %0 to <25 x i32>3109  %2 = load <25 x i8>, ptr %b3110  %3 = sext <25 x i8> %2 to <25 x i32>3111  %4 = mul nsw <25 x i32> %3, %13112  %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4)3113  %op.extra = add nsw i32 %5, %sum3114  ret i32 %op.extra3115}3116 3117define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {3118; CHECK-SD-LABEL: test_sdot_v25i8_double:3119; CHECK-SD:       // %bb.0: // %entry3120; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill3121; CHECK-SD-NEXT:    .cfi_def_cfa_offset 163122; CHECK-SD-NEXT:    .cfi_offset w29, -163123; CHECK-SD-NEXT:    ldr b0, [sp, #216]3124; CHECK-SD-NEXT:    add x8, sp, #2243125; CHECK-SD-NEXT:    ldr b1, [sp, #16]3126; CHECK-SD-NEXT:    ldr b2, [sp, #280]3127; CHECK-SD-NEXT:    add x9, sp, #2403128; CHECK-SD-NEXT:    ldr b4, [sp, #80]3129; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]3130; CHECK-SD-NEXT:    add x8, sp, #243131; CHECK-SD-NEXT:    add x10, sp, #483132; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x8]3133; CHECK-SD-NEXT:    add x8, sp, #2323134; CHECK-SD-NEXT:    add x11, sp, #963135; CHECK-SD-NEXT:    ldr b5, [sp, #152]3136; CHECK-SD-NEXT:    add x12, sp, #1683137; CHECK-SD-NEXT:    ldr b6, [sp, #616]3138; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]3139; CHECK-SD-NEXT:    add x8, sp, #323140; CHECK-SD-NEXT:    fmov s3, w03141; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x8]3142; CHECK-SD-NEXT:    add x8, sp, #2883143; CHECK-SD-NEXT:    ldr b7, [sp, #416]3144; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x8]3145; CHECK-SD-NEXT:    add x8, sp, #403146; CHECK-SD-NEXT:    ldr b22, [sp, #744]3147; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x9]3148; CHECK-SD-NEXT:    add x9, sp, #2483149; CHECK-SD-NEXT:    mov v3.b[1], w13150; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]3151; CHECK-SD-NEXT:    add x8, sp, #883152; CHECK-SD-NEXT:    ldr b23, [sp, #544]3153; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x8]3154; CHECK-SD-NEXT:    add x8, sp, #2563155; CHECK-SD-NEXT:    ldr b19, [sp, #680]3156; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x9]3157; CHECK-SD-NEXT:    add x9, sp, #2963158; CHECK-SD-NEXT:    ldr b20, [sp, #480]3159; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x10]3160; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x9]3161; CHECK-SD-NEXT:    add x10, sp, #1603162; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x11]3163; CHECK-SD-NEXT:    add x11, sp, #3043164; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x10]3165; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x8]3166; CHECK-SD-NEXT:    add x8, sp, #563167; CHECK-SD-NEXT:    add x10, sp, #2643168; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x8]3169; CHECK-SD-NEXT:    add x8, sp, #643170; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x11]3171; CHECK-SD-NEXT:    add x9, sp, #2723172; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x12]3173; CHECK-SD-NEXT:    add x11, sp, #723174; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x10]3175; CHECK-SD-NEXT:    add x10, sp, #3123176; CHECK-SD-NEXT:    mov v3.b[2], w23177; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x8]3178; CHECK-SD-NEXT:    add x8, sp, #1043179; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x10]3180; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x8]3181; CHECK-SD-NEXT:    add x8, sp, #1123182; CHECK-SD-NEXT:    add x10, sp, #1283183; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x9]3184; CHECK-SD-NEXT:    add x9, sp, #3203185; CHECK-SD-NEXT:    ldr b21, [sp, #552]3186; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x9]3187; CHECK-SD-NEXT:    add x9, sp, #1763188; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x11]3189; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]3190; CHECK-SD-NEXT:    add x8, sp, #6243191; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x9]3192; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x8]3193; CHECK-SD-NEXT:    add x8, sp, #1203194; CHECK-SD-NEXT:    add x9, sp, #3283195; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x9]3196; CHECK-SD-NEXT:    add x9, sp, #1843197; CHECK-SD-NEXT:    add x11, sp, #1923198; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x8]3199; CHECK-SD-NEXT:    add x8, sp, #6323200; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x9]3201; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]3202; CHECK-SD-NEXT:    add x9, sp, #6403203; CHECK-SD-NEXT:    add x8, sp, #3363204; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x8]3205; CHECK-SD-NEXT:    add x8, sp, #6563206; CHECK-SD-NEXT:    smull v23.8h, v23.8b, v22.8b3207; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x11]3208; CHECK-SD-NEXT:    add x11, sp, #6483209; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x10]3210; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x9]3211; CHECK-SD-NEXT:    add x9, sp, #2003212; CHECK-SD-NEXT:    add x10, sp, #1363213; CHECK-SD-NEXT:    ldr b22, [sp, #352]3214; CHECK-SD-NEXT:    add x12, sp, #3603215; CHECK-SD-NEXT:    mov v3.b[3], w33216; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x9]3217; CHECK-SD-NEXT:    add x9, sp, #2083218; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x10]3219; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x11]3220; CHECK-SD-NEXT:    add x11, sp, #4243221; CHECK-SD-NEXT:    add x10, sp, #4883222; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x11]3223; CHECK-SD-NEXT:    add x11, sp, #5603224; CHECK-SD-NEXT:    ld1 { v20.b }[1], [x10]3225; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x9]3226; CHECK-SD-NEXT:    add x9, sp, #4403227; CHECK-SD-NEXT:    ld1 { v21.b }[1], [x11]3228; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x8]3229; CHECK-SD-NEXT:    add x8, sp, #4323230; CHECK-SD-NEXT:    ld1 { v22.b }[1], [x12]3231; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x8]3232; CHECK-SD-NEXT:    add x11, sp, #4963233; CHECK-SD-NEXT:    add x12, sp, #5683234; CHECK-SD-NEXT:    add x13, sp, #3683235; CHECK-SD-NEXT:    ld1 { v20.b }[2], [x11]3236; CHECK-SD-NEXT:    ld1 { v21.b }[2], [x12]3237; CHECK-SD-NEXT:    ld1 { v22.b }[2], [x13]3238; CHECK-SD-NEXT:    add x10, sp, #4483239; CHECK-SD-NEXT:    mov v3.b[4], w43240; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]3241; CHECK-SD-NEXT:    add x9, sp, #6883242; CHECK-SD-NEXT:    add x11, sp, #5763243; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x9]3244; CHECK-SD-NEXT:    add x9, sp, #6963245; CHECK-SD-NEXT:    add x12, sp, #3763246; CHECK-SD-NEXT:    ld1 { v21.b }[3], [x11]3247; CHECK-SD-NEXT:    ld1 { v22.b }[3], [x12]3248; CHECK-SD-NEXT:    add x11, sp, #5123249; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]3250; CHECK-SD-NEXT:    add x10, sp, #5043251; CHECK-SD-NEXT:    add x12, sp, #5843252; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x9]3253; CHECK-SD-NEXT:    add x9, sp, #7043254; CHECK-SD-NEXT:    ld1 { v20.b }[3], [x10]3255; CHECK-SD-NEXT:    add x13, sp, #3843256; CHECK-SD-NEXT:    mov v3.b[5], w53257; CHECK-SD-NEXT:    ld1 { v21.b }[4], [x12]3258; CHECK-SD-NEXT:    ld1 { v22.b }[4], [x13]3259; CHECK-SD-NEXT:    add x10, sp, #4563260; CHECK-SD-NEXT:    ldr b16, [sp, #344]3261; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x9]3262; CHECK-SD-NEXT:    add x9, sp, #7123263; CHECK-SD-NEXT:    ld1 { v20.b }[4], [x11]3264; CHECK-SD-NEXT:    ldr b17, [sp, #144]3265; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x10]3266; CHECK-SD-NEXT:    add x10, sp, #5203267; CHECK-SD-NEXT:    add x11, sp, #5923268; CHECK-SD-NEXT:    add x12, sp, #3923269; CHECK-SD-NEXT:    mov v3.b[6], w63270; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x9]3271; CHECK-SD-NEXT:    add x9, sp, #7203272; CHECK-SD-NEXT:    ld1 { v20.b }[5], [x10]3273; CHECK-SD-NEXT:    ld1 { v21.b }[5], [x11]3274; CHECK-SD-NEXT:    ld1 { v22.b }[5], [x12]3275; CHECK-SD-NEXT:    smull v16.8h, v17.8b, v16.8b3276; CHECK-SD-NEXT:    add x8, sp, #6643277; CHECK-SD-NEXT:    add x10, sp, #4643278; CHECK-SD-NEXT:    add x11, sp, #5283279; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x9]3280; CHECK-SD-NEXT:    add x9, sp, #7283281; CHECK-SD-NEXT:    add x12, sp, #6003282; CHECK-SD-NEXT:    add x13, sp, #4003283; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x8]3284; CHECK-SD-NEXT:    ld1 { v20.b }[6], [x11]3285; CHECK-SD-NEXT:    ld1 { v21.b }[6], [x12]3286; CHECK-SD-NEXT:    ld1 { v22.b }[6], [x13]3287; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x10]3288; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x9]3289; CHECK-SD-NEXT:    add x9, sp, #7363290; CHECK-SD-NEXT:    mov v3.b[7], w73291; CHECK-SD-NEXT:    sshll v18.4s, v16.4h, #03292; CHECK-SD-NEXT:    movi v16.2d, #00000000000000003293; CHECK-SD-NEXT:    movi v17.2d, #00000000000000003294; CHECK-SD-NEXT:    add x8, sp, #6723295; CHECK-SD-NEXT:    add x10, sp, #4723296; CHECK-SD-NEXT:    add x11, sp, #6083297; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x9]3298; CHECK-SD-NEXT:    add x9, sp, #5363299; CHECK-SD-NEXT:    add x12, sp, #4083300; CHECK-SD-NEXT:    ld1 { v20.b }[7], [x9]3301; CHECK-SD-NEXT:    ld1 { v21.b }[7], [x11]3302; CHECK-SD-NEXT:    ld1 { v22.b }[7], [x12]3303; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x8]3304; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x10]3305; CHECK-SD-NEXT:    sshll v23.4s, v23.4h, #03306; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b3307; CHECK-SD-NEXT:    smull v1.8h, v4.8b, v2.8b3308; CHECK-SD-NEXT:    smull v2.8h, v3.8b, v5.8b3309; CHECK-SD-NEXT:    smull v3.8h, v20.8b, v19.8b3310; CHECK-SD-NEXT:    smull v4.8h, v22.8b, v21.8b3311; CHECK-SD-NEXT:    mov v17.s[0], v18.s[0]3312; CHECK-SD-NEXT:    smull v5.8h, v7.8b, v6.8b3313; CHECK-SD-NEXT:    mov v16.s[0], v23.s[0]3314; CHECK-SD-NEXT:    saddl2 v6.4s, v2.8h, v1.8h3315; CHECK-SD-NEXT:    saddl v1.4s, v2.4h, v1.4h3316; CHECK-SD-NEXT:    saddl2 v2.4s, v4.8h, v3.8h3317; CHECK-SD-NEXT:    saddl v3.4s, v4.4h, v3.4h3318; CHECK-SD-NEXT:    saddw v4.4s, v17.4s, v0.4h3319; CHECK-SD-NEXT:    saddw v7.4s, v16.4s, v5.4h3320; CHECK-SD-NEXT:    saddw2 v0.4s, v6.4s, v0.8h3321; CHECK-SD-NEXT:    add v1.4s, v1.4s, v4.4s3322; CHECK-SD-NEXT:    saddw2 v2.4s, v2.4s, v5.8h3323; CHECK-SD-NEXT:    add v3.4s, v3.4s, v7.4s3324; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s3325; CHECK-SD-NEXT:    add v1.4s, v3.4s, v2.4s3326; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s3327; CHECK-SD-NEXT:    addv s0, v0.4s3328; CHECK-SD-NEXT:    fmov w0, s03329; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload3330; CHECK-SD-NEXT:    ret3331;3332; CHECK-GI-LABEL: test_sdot_v25i8_double:3333; CHECK-GI:       // %bb.0: // %entry3334; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill3335; CHECK-GI-NEXT:    .cfi_def_cfa_offset 163336; CHECK-GI-NEXT:    .cfi_offset w29, -163337; CHECK-GI-NEXT:    lsl w8, w0, #83338; CHECK-GI-NEXT:    ldr w9, [sp, #16]3339; CHECK-GI-NEXT:    lsl w10, w1, #83340; CHECK-GI-NEXT:    ldr w11, [sp, #24]3341; CHECK-GI-NEXT:    lsl w12, w4, #83342; CHECK-GI-NEXT:    ldr w13, [sp, #56]3343; CHECK-GI-NEXT:    sbfx w8, w8, #8, #83344; CHECK-GI-NEXT:    lsl w9, w9, #83345; CHECK-GI-NEXT:    sbfx w10, w10, #8, #83346; CHECK-GI-NEXT:    sbfx w12, w12, #8, #83347; CHECK-GI-NEXT:    lsl w13, w13, #83348; CHECK-GI-NEXT:    ldr w14, [sp, #64]3349; CHECK-GI-NEXT:    fmov s2, w83350; CHECK-GI-NEXT:    sbfx w9, w9, #8, #83351; CHECK-GI-NEXT:    lsl w8, w11, #83352; CHECK-GI-NEXT:    lsl w11, w2, #83353; CHECK-GI-NEXT:    sbfx w13, w13, #8, #83354; CHECK-GI-NEXT:    lsl w14, w14, #83355; CHECK-GI-NEXT:    fmov s4, w93356; CHECK-GI-NEXT:    sbfx w8, w8, #8, #83357; CHECK-GI-NEXT:    ldr w16, [sp, #112]3358; CHECK-GI-NEXT:    mov v2.h[1], w103359; CHECK-GI-NEXT:    ldr w10, [sp, #32]3360; CHECK-GI-NEXT:    sbfx w9, w11, #8, #83361; CHECK-GI-NEXT:    lsl w11, w3, #83362; CHECK-GI-NEXT:    sbfx w14, w14, #8, #83363; CHECK-GI-NEXT:    movi d1, #00000000000000003364; CHECK-GI-NEXT:    lsl w10, w10, #83365; CHECK-GI-NEXT:    mov v4.h[1], w83366; CHECK-GI-NEXT:    ldr w8, [sp, #152]3367; CHECK-GI-NEXT:    sbfx w11, w11, #8, #83368; CHECK-GI-NEXT:    movi d0, #00000000000000003369; CHECK-GI-NEXT:    mov v2.h[2], w93370; CHECK-GI-NEXT:    ldr w9, [sp, #40]3371; CHECK-GI-NEXT:    sbfx w10, w10, #8, #83372; CHECK-GI-NEXT:    lsl w8, w8, #83373; CHECK-GI-NEXT:    mov v1.s[1], wzr3374; CHECK-GI-NEXT:    mov v4.h[2], w103375; CHECK-GI-NEXT:    lsl w9, w9, #83376; CHECK-GI-NEXT:    ldr w10, [sp, #160]3377; CHECK-GI-NEXT:    sbfx w8, w8, #8, #83378; CHECK-GI-NEXT:    mov v0.s[1], wzr3379; CHECK-GI-NEXT:    mov v2.h[3], w113380; CHECK-GI-NEXT:    sbfx w9, w9, #8, #83381; CHECK-GI-NEXT:    ldr w11, [sp, #48]3382; CHECK-GI-NEXT:    lsl w10, w10, #83383; CHECK-GI-NEXT:    fmov s3, w83384; CHECK-GI-NEXT:    mov v1.s[2], wzr3385; CHECK-GI-NEXT:    mov v4.h[3], w93386; CHECK-GI-NEXT:    ldr w9, [sp, #80]3387; CHECK-GI-NEXT:    lsl w8, w11, #83388; CHECK-GI-NEXT:    sbfx w10, w10, #8, #83389; CHECK-GI-NEXT:    ldr w11, [sp, #168]3390; CHECK-GI-NEXT:    mov v0.s[2], wzr3391; CHECK-GI-NEXT:    mov v2.h[4], w123392; CHECK-GI-NEXT:    lsl w12, w5, #83393; CHECK-GI-NEXT:    lsl w9, w9, #83394; CHECK-GI-NEXT:    sbfx w8, w8, #8, #83395; CHECK-GI-NEXT:    mov v3.h[1], w103396; CHECK-GI-NEXT:    ldr w10, [sp, #88]3397; CHECK-GI-NEXT:    lsl w11, w11, #83398; CHECK-GI-NEXT:    sbfx w12, w12, #8, #83399; CHECK-GI-NEXT:    sbfx w9, w9, #8, #83400; CHECK-GI-NEXT:    mov v4.h[4], w83401; CHECK-GI-NEXT:    lsl w8, w10, #83402; CHECK-GI-NEXT:    ldr w10, [sp, #176]3403; CHECK-GI-NEXT:    mov v2.h[5], w123404; CHECK-GI-NEXT:    sbfx w11, w11, #8, #83405; CHECK-GI-NEXT:    lsl w12, w6, #83406; CHECK-GI-NEXT:    fmov s6, w93407; CHECK-GI-NEXT:    sbfx w15, w8, #8, #83408; CHECK-GI-NEXT:    lsl w9, w10, #83409; CHECK-GI-NEXT:    mov v3.h[2], w113410; CHECK-GI-NEXT:    sbfx w11, w12, #8, #83411; CHECK-GI-NEXT:    ldr w10, [sp, #96]3412; CHECK-GI-NEXT:    sbfx w9, w9, #8, #83413; CHECK-GI-NEXT:    mov v4.h[5], w133414; CHECK-GI-NEXT:    ldr w13, [sp, #224]3415; CHECK-GI-NEXT:    mov v6.h[1], w153416; CHECK-GI-NEXT:    mov v2.h[6], w113417; CHECK-GI-NEXT:    lsl w15, w7, #83418; CHECK-GI-NEXT:    lsl w10, w10, #83419; CHECK-GI-NEXT:    ldr w11, [sp, #184]3420; CHECK-GI-NEXT:    ldr w12, [sp, #104]3421; CHECK-GI-NEXT:    mov v3.h[3], w93422; CHECK-GI-NEXT:    ldr w9, [sp, #216]3423; CHECK-GI-NEXT:    sbfx w15, w15, #8, #83424; CHECK-GI-NEXT:    sbfx w10, w10, #8, #83425; CHECK-GI-NEXT:    lsl w11, w11, #83426; CHECK-GI-NEXT:    lsl w12, w12, #83427; CHECK-GI-NEXT:    mov v2.h[7], w153428; CHECK-GI-NEXT:    lsl w15, w9, #83429; CHECK-GI-NEXT:    mov v4.h[6], w143430; CHECK-GI-NEXT:    mov v6.h[2], w103431; CHECK-GI-NEXT:    lsl w10, w13, #83432; CHECK-GI-NEXT:    sbfx w11, w11, #8, #83433; CHECK-GI-NEXT:    sbfx w13, w15, #8, #83434; CHECK-GI-NEXT:    sbfx w12, w12, #8, #83435; CHECK-GI-NEXT:    ldr w14, [sp, #288]3436; CHECK-GI-NEXT:    sbfx w10, w10, #8, #83437; CHECK-GI-NEXT:    mov v3.h[4], w113438; CHECK-GI-NEXT:    ldr w11, [sp, #192]3439; CHECK-GI-NEXT:    fmov s5, w133440; CHECK-GI-NEXT:    ldr w13, [sp, #232]3441; CHECK-GI-NEXT:    ldr w9, [sp, #120]3442; CHECK-GI-NEXT:    lsl w11, w11, #83443; CHECK-GI-NEXT:    mov v6.h[3], w123444; CHECK-GI-NEXT:    ldr w8, [sp, #72]3445; CHECK-GI-NEXT:    lsl w13, w13, #83446; CHECK-GI-NEXT:    lsl w9, w9, #83447; CHECK-GI-NEXT:    mov v1.s[3], wzr3448; CHECK-GI-NEXT:    mov v5.h[1], w103449; CHECK-GI-NEXT:    ldr w10, [sp, #280]3450; CHECK-GI-NEXT:    sbfx w15, w11, #8, #83451; CHECK-GI-NEXT:    sbfx w12, w13, #8, #83452; CHECK-GI-NEXT:    lsl w13, w14, #83453; CHECK-GI-NEXT:    ldr w14, [sp, #240]3454; CHECK-GI-NEXT:    lsl w10, w10, #83455; CHECK-GI-NEXT:    mov v3.h[5], w153456; CHECK-GI-NEXT:    lsl w15, w16, #83457; CHECK-GI-NEXT:    lsl w14, w14, #83458; CHECK-GI-NEXT:    sbfx w13, w13, #8, #83459; CHECK-GI-NEXT:    sbfx w9, w9, #8, #83460; CHECK-GI-NEXT:    sbfx w10, w10, #8, #83461; CHECK-GI-NEXT:    mov v5.h[2], w123462; CHECK-GI-NEXT:    ldr w12, [sp, #296]3463; CHECK-GI-NEXT:    sbfx w14, w14, #8, #83464; CHECK-GI-NEXT:    sbfx w15, w15, #8, #83465; CHECK-GI-NEXT:    lsl w8, w8, #83466; CHECK-GI-NEXT:    fmov s7, w103467; CHECK-GI-NEXT:    ldr w10, [sp, #200]3468; CHECK-GI-NEXT:    lsl w12, w12, #83469; CHECK-GI-NEXT:    mov v6.h[4], w153470; CHECK-GI-NEXT:    ldr w15, [sp, #304]3471; CHECK-GI-NEXT:    ldr w11, [sp, #128]3472; CHECK-GI-NEXT:    lsl w10, w10, #83473; CHECK-GI-NEXT:    mov v5.h[3], w143474; CHECK-GI-NEXT:    ldr w14, [sp, #208]3475; CHECK-GI-NEXT:    mov v7.h[1], w133476; CHECK-GI-NEXT:    ldr w13, [sp, #248]3477; CHECK-GI-NEXT:    sbfx w12, w12, #8, #83478; CHECK-GI-NEXT:    sbfx w10, w10, #8, #83479; CHECK-GI-NEXT:    lsl w15, w15, #83480; CHECK-GI-NEXT:    sbfx w8, w8, #8, #83481; CHECK-GI-NEXT:    lsl w13, w13, #83482; CHECK-GI-NEXT:    mov v6.h[5], w93483; CHECK-GI-NEXT:    ldr w9, [sp, #272]3484; CHECK-GI-NEXT:    mov v3.h[6], w103485; CHECK-GI-NEXT:    lsl w10, w14, #83486; CHECK-GI-NEXT:    sbfx w14, w15, #8, #83487; CHECK-GI-NEXT:    mov v7.h[2], w123488; CHECK-GI-NEXT:    ldr w12, [sp, #256]3489; CHECK-GI-NEXT:    sbfx w13, w13, #8, #83490; CHECK-GI-NEXT:    sbfx w10, w10, #8, #83491; CHECK-GI-NEXT:    ldr w15, [sp, #320]3492; CHECK-GI-NEXT:    lsl w9, w9, #83493; CHECK-GI-NEXT:    mov v5.h[4], w133494; CHECK-GI-NEXT:    lsl w12, w12, #83495; CHECK-GI-NEXT:    ldr w13, [sp, #312]3496; CHECK-GI-NEXT:    mov v3.h[7], w103497; CHECK-GI-NEXT:    lsl w11, w11, #83498; CHECK-GI-NEXT:    mov v4.h[7], w83499; CHECK-GI-NEXT:    mov v7.h[3], w143500; CHECK-GI-NEXT:    ldr w14, [sp, #264]3501; CHECK-GI-NEXT:    sbfx w12, w12, #8, #83502; CHECK-GI-NEXT:    lsl w13, w13, #83503; CHECK-GI-NEXT:    sbfx w8, w9, #8, #83504; CHECK-GI-NEXT:    ldr w16, [sp, #136]3505; CHECK-GI-NEXT:    lsl w14, w14, #83506; CHECK-GI-NEXT:    mov v5.h[5], w123507; CHECK-GI-NEXT:    sbfx w11, w11, #8, #83508; CHECK-GI-NEXT:    mul v16.8h, v2.8h, v3.8h3509; CHECK-GI-NEXT:    sbfx w13, w13, #8, #83510; CHECK-GI-NEXT:    lsl w16, w16, #83511; CHECK-GI-NEXT:    sbfx w12, w14, #8, #83512; CHECK-GI-NEXT:    lsl w14, w15, #83513; CHECK-GI-NEXT:    mov v6.h[6], w113514; CHECK-GI-NEXT:    mov v7.h[4], w133515; CHECK-GI-NEXT:    ldr w13, [sp, #328]3516; CHECK-GI-NEXT:    ldr w10, [sp, #144]3517; CHECK-GI-NEXT:    mov v5.h[6], w123518; CHECK-GI-NEXT:    ldr w12, [sp, #336]3519; CHECK-GI-NEXT:    sbfx w14, w14, #8, #83520; CHECK-GI-NEXT:    smov w9, v16.h[0]3521; CHECK-GI-NEXT:    smov w15, v16.h[4]3522; CHECK-GI-NEXT:    lsl w13, w13, #83523; CHECK-GI-NEXT:    smov w17, v16.h[5]3524; CHECK-GI-NEXT:    sxtb w10, w103525; CHECK-GI-NEXT:    mov v0.s[3], wzr3526; CHECK-GI-NEXT:    sbfx w11, w13, #8, #83527; CHECK-GI-NEXT:    smov w13, v16.h[1]3528; CHECK-GI-NEXT:    mov v7.h[5], w143529; CHECK-GI-NEXT:    mov v5.h[7], w83530; CHECK-GI-NEXT:    ldr w14, [sp, #344]3531; CHECK-GI-NEXT:    ldr w8, [sp, #352]3532; CHECK-GI-NEXT:    fmov s2, w93533; CHECK-GI-NEXT:    fmov s3, w153534; CHECK-GI-NEXT:    lsl w9, w12, #83535; CHECK-GI-NEXT:    sbfx w12, w16, #8, #83536; CHECK-GI-NEXT:    sxtb w14, w143537; CHECK-GI-NEXT:    lsl w8, w8, #83538; CHECK-GI-NEXT:    mov v7.h[6], w113539; CHECK-GI-NEXT:    ldr w11, [sp, #360]3540; CHECK-GI-NEXT:    smov w15, v16.h[3]3541; CHECK-GI-NEXT:    mov v2.s[1], w133542; CHECK-GI-NEXT:    smov w13, v16.h[2]3543; CHECK-GI-NEXT:    mov v6.h[7], w123544; CHECK-GI-NEXT:    smov w12, v16.h[6]3545; CHECK-GI-NEXT:    mov v3.s[1], w173546; CHECK-GI-NEXT:    mul v18.8h, v4.8h, v5.8h3547; CHECK-GI-NEXT:    lsl w11, w11, #83548; CHECK-GI-NEXT:    sbfx w16, w9, #8, #83549; CHECK-GI-NEXT:    ldr w9, [sp, #368]3550; CHECK-GI-NEXT:    mov v2.s[2], w133551; CHECK-GI-NEXT:    smov w13, v16.h[7]3552; CHECK-GI-NEXT:    sbfx w11, w11, #8, #83553; CHECK-GI-NEXT:    mov v3.s[2], w123554; CHECK-GI-NEXT:    sbfx w12, w8, #8, #83555; CHECK-GI-NEXT:    mul w8, w10, w143556; CHECK-GI-NEXT:    smov w10, v18.h[0]3557; CHECK-GI-NEXT:    lsl w9, w9, #83558; CHECK-GI-NEXT:    ldr w14, [sp, #376]3559; CHECK-GI-NEXT:    fmov s16, w123560; CHECK-GI-NEXT:    smov w12, v18.h[1]3561; CHECK-GI-NEXT:    mov v7.h[7], w163562; CHECK-GI-NEXT:    mov v2.s[3], w153563; CHECK-GI-NEXT:    smov w15, v18.h[4]3564; CHECK-GI-NEXT:    sbfx w9, w9, #8, #83565; CHECK-GI-NEXT:    mov v3.s[3], w133566; CHECK-GI-NEXT:    ldr w13, [sp, #416]3567; CHECK-GI-NEXT:    lsl w14, w14, #83568; CHECK-GI-NEXT:    fmov s4, w103569; CHECK-GI-NEXT:    mov v16.h[1], w113570; CHECK-GI-NEXT:    ldr w10, [sp, #424]3571; CHECK-GI-NEXT:    lsl w13, w13, #83572; CHECK-GI-NEXT:    ldr w11, [sp, #384]3573; CHECK-GI-NEXT:    sbfx w14, w14, #8, #83574; CHECK-GI-NEXT:    fmov s5, w153575; CHECK-GI-NEXT:    lsl w10, w10, #83576; CHECK-GI-NEXT:    ldr w15, [sp, #432]3577; CHECK-GI-NEXT:    mov v4.s[1], w123578; CHECK-GI-NEXT:    smov w12, v18.h[5]3579; CHECK-GI-NEXT:    sbfx w13, w13, #8, #83580; CHECK-GI-NEXT:    mov v16.h[2], w93581; CHECK-GI-NEXT:    sbfx w10, w10, #8, #83582; CHECK-GI-NEXT:    lsl w15, w15, #83583; CHECK-GI-NEXT:    fmov s17, w133584; CHECK-GI-NEXT:    lsl w11, w11, #83585; CHECK-GI-NEXT:    mul v7.8h, v6.8h, v7.8h3586; CHECK-GI-NEXT:    sbfx w15, w15, #8, #83587; CHECK-GI-NEXT:    ldr w9, [sp, #392]3588; CHECK-GI-NEXT:    ldr w13, [sp, #400]3589; CHECK-GI-NEXT:    mov v5.s[1], w123590; CHECK-GI-NEXT:    smov w12, v18.h[2]3591; CHECK-GI-NEXT:    sbfx w11, w11, #8, #83592; CHECK-GI-NEXT:    mov v17.h[1], w103593; CHECK-GI-NEXT:    mov v16.h[3], w143594; CHECK-GI-NEXT:    ldr w10, [sp, #440]3595; CHECK-GI-NEXT:    smov w14, v18.h[6]3596; CHECK-GI-NEXT:    lsl w9, w9, #83597; CHECK-GI-NEXT:    ldr w16, [sp, #456]3598; CHECK-GI-NEXT:    lsl w10, w10, #83599; CHECK-GI-NEXT:    sxth w8, w83600; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s3601; CHECK-GI-NEXT:    mov v4.s[2], w123602; CHECK-GI-NEXT:    smov w12, v18.h[3]3603; CHECK-GI-NEXT:    sbfx w9, w9, #8, #83604; CHECK-GI-NEXT:    mov v17.h[2], w153605; CHECK-GI-NEXT:    mov v16.h[4], w113606; CHECK-GI-NEXT:    sbfx w10, w10, #8, #83607; CHECK-GI-NEXT:    mov v5.s[2], w143608; CHECK-GI-NEXT:    smov w14, v18.h[7]3609; CHECK-GI-NEXT:    ldr w15, [sp, #448]3610; CHECK-GI-NEXT:    ldr w11, [sp, #408]3611; CHECK-GI-NEXT:    mov v4.s[3], w123612; CHECK-GI-NEXT:    smov w12, v7.h[0]3613; CHECK-GI-NEXT:    mov v17.h[3], w103614; CHECK-GI-NEXT:    ldr w10, [sp, #480]3615; CHECK-GI-NEXT:    mov v16.h[5], w93616; CHECK-GI-NEXT:    lsl w9, w13, #83617; CHECK-GI-NEXT:    lsl w13, w15, #83618; CHECK-GI-NEXT:    mov v5.s[3], w143619; CHECK-GI-NEXT:    lsl w10, w10, #83620; CHECK-GI-NEXT:    smov w14, v7.h[1]3621; CHECK-GI-NEXT:    lsl w15, w16, #83622; CHECK-GI-NEXT:    fmov s6, w123623; CHECK-GI-NEXT:    ldr w12, [sp, #488]3624; CHECK-GI-NEXT:    sbfx w13, w13, #8, #83625; CHECK-GI-NEXT:    sbfx w10, w10, #8, #83626; CHECK-GI-NEXT:    sbfx w9, w9, #8, #83627; CHECK-GI-NEXT:    sbfx w15, w15, #8, #83628; CHECK-GI-NEXT:    lsl w12, w12, #83629; CHECK-GI-NEXT:    mov v17.h[4], w133630; CHECK-GI-NEXT:    ldr w13, [sp, #496]3631; CHECK-GI-NEXT:    fmov s18, w103632; CHECK-GI-NEXT:    ldr w10, [sp, #552]3633; CHECK-GI-NEXT:    mov v6.s[1], w143634; CHECK-GI-NEXT:    sbfx w12, w12, #8, #83635; CHECK-GI-NEXT:    ldr w14, [sp, #464]3636; CHECK-GI-NEXT:    mov v16.h[6], w93637; CHECK-GI-NEXT:    lsl w10, w10, #83638; CHECK-GI-NEXT:    lsl w11, w11, #83639; CHECK-GI-NEXT:    lsl w13, w13, #83640; CHECK-GI-NEXT:    mov v18.h[1], w123641; CHECK-GI-NEXT:    ldr w12, [sp, #560]3642; CHECK-GI-NEXT:    mov v17.h[5], w153643; CHECK-GI-NEXT:    sbfx w15, w10, #8, #83644; CHECK-GI-NEXT:    lsl w14, w14, #83645; CHECK-GI-NEXT:    sbfx w11, w11, #8, #83646; CHECK-GI-NEXT:    sbfx w13, w13, #8, #83647; CHECK-GI-NEXT:    lsl w12, w12, #83648; CHECK-GI-NEXT:    ldr w10, [sp, #512]3649; CHECK-GI-NEXT:    fmov s19, w153650; CHECK-GI-NEXT:    ldr w15, [sp, #616]3651; CHECK-GI-NEXT:    sbfx w14, w14, #8, #83652; CHECK-GI-NEXT:    mov v16.h[7], w113653; CHECK-GI-NEXT:    ldr w11, [sp, #504]3654; CHECK-GI-NEXT:    mov v18.h[2], w133655; CHECK-GI-NEXT:    ldr w13, [sp, #568]3656; CHECK-GI-NEXT:    sbfx w12, w12, #8, #83657; CHECK-GI-NEXT:    mov v17.h[6], w143658; CHECK-GI-NEXT:    lsl w14, w15, #83659; CHECK-GI-NEXT:    lsl w11, w11, #83660; CHECK-GI-NEXT:    ldr w15, [sp, #576]3661; CHECK-GI-NEXT:    mov v19.h[1], w123662; CHECK-GI-NEXT:    ldr w12, [sp, #624]3663; CHECK-GI-NEXT:    lsl w13, w13, #83664; CHECK-GI-NEXT:    sbfx w14, w14, #8, #83665; CHECK-GI-NEXT:    sbfx w16, w11, #8, #83666; CHECK-GI-NEXT:    lsl w10, w10, #83667; CHECK-GI-NEXT:    sbfx w13, w13, #8, #83668; CHECK-GI-NEXT:    lsl w12, w12, #83669; CHECK-GI-NEXT:    lsl w15, w15, #83670; CHECK-GI-NEXT:    fmov s20, w143671; CHECK-GI-NEXT:    ldr w14, [sp, #680]3672; CHECK-GI-NEXT:    mov v18.h[3], w163673; CHECK-GI-NEXT:    sbfx w12, w12, #8, #83674; CHECK-GI-NEXT:    mov v19.h[2], w133675; CHECK-GI-NEXT:    ldr w13, [sp, #632]3676; CHECK-GI-NEXT:    lsl w14, w14, #83677; CHECK-GI-NEXT:    sbfx w10, w10, #8, #83678; CHECK-GI-NEXT:    sbfx w15, w15, #8, #83679; CHECK-GI-NEXT:    mov v20.h[1], w123680; CHECK-GI-NEXT:    ldr w12, [sp, #688]3681; CHECK-GI-NEXT:    lsl w13, w13, #83682; CHECK-GI-NEXT:    sbfx w14, w14, #8, #83683; CHECK-GI-NEXT:    mov v18.h[4], w103684; CHECK-GI-NEXT:    ldr w10, [sp, #584]3685; CHECK-GI-NEXT:    sbfx w13, w13, #8, #83686; CHECK-GI-NEXT:    lsl w12, w12, #83687; CHECK-GI-NEXT:    mov v19.h[3], w153688; CHECK-GI-NEXT:    fmov s21, w143689; CHECK-GI-NEXT:    ldr w15, [sp, #640]3690; CHECK-GI-NEXT:    lsl w10, w10, #83691; CHECK-GI-NEXT:    mov v20.h[2], w133692; CHECK-GI-NEXT:    ldr w13, [sp, #696]3693; CHECK-GI-NEXT:    sbfx w12, w12, #8, #83694; CHECK-GI-NEXT:    ldr w11, [sp, #520]3695; CHECK-GI-NEXT:    sbfx w10, w10, #8, #83696; CHECK-GI-NEXT:    lsl w15, w15, #83697; CHECK-GI-NEXT:    lsl w13, w13, #83698; CHECK-GI-NEXT:    mov v21.h[1], w123699; CHECK-GI-NEXT:    ldr w12, [sp, #592]3700; CHECK-GI-NEXT:    sbfx w15, w15, #8, #83701; CHECK-GI-NEXT:    mov v19.h[4], w103702; CHECK-GI-NEXT:    ldr w10, [sp, #704]3703; CHECK-GI-NEXT:    lsl w11, w11, #83704; CHECK-GI-NEXT:    sbfx w13, w13, #8, #83705; CHECK-GI-NEXT:    lsl w12, w12, #83706; CHECK-GI-NEXT:    mov v20.h[3], w153707; CHECK-GI-NEXT:    ldr w15, [sp, #648]3708; CHECK-GI-NEXT:    lsl w10, w10, #83709; CHECK-GI-NEXT:    sbfx w11, w11, #8, #83710; CHECK-GI-NEXT:    mov v21.h[2], w133711; CHECK-GI-NEXT:    ldr w13, [sp, #600]3712; CHECK-GI-NEXT:    sbfx w12, w12, #8, #83713; CHECK-GI-NEXT:    lsl w15, w15, #83714; CHECK-GI-NEXT:    sbfx w10, w10, #8, #83715; CHECK-GI-NEXT:    mov v18.h[5], w113716; CHECK-GI-NEXT:    ldr w11, [sp, #712]3717; CHECK-GI-NEXT:    lsl w13, w13, #83718; CHECK-GI-NEXT:    mov v19.h[5], w123719; CHECK-GI-NEXT:    sbfx w15, w15, #8, #83720; CHECK-GI-NEXT:    ldr w12, [sp, #656]3721; CHECK-GI-NEXT:    lsl w11, w11, #83722; CHECK-GI-NEXT:    mov v21.h[3], w103723; CHECK-GI-NEXT:    sbfx w13, w13, #8, #83724; CHECK-GI-NEXT:    ldr w10, [sp, #608]3725; CHECK-GI-NEXT:    mov v20.h[4], w153726; CHECK-GI-NEXT:    lsl w12, w12, #83727; CHECK-GI-NEXT:    sbfx w11, w11, #8, #83728; CHECK-GI-NEXT:    ldr w14, [sp, #528]3729; CHECK-GI-NEXT:    ldr w15, [sp, #664]3730; CHECK-GI-NEXT:    mov v19.h[6], w133731; CHECK-GI-NEXT:    ldr w13, [sp, #720]3732; CHECK-GI-NEXT:    lsl w10, w10, #83733; CHECK-GI-NEXT:    sbfx w12, w12, #8, #83734; CHECK-GI-NEXT:    mov v21.h[4], w113735; CHECK-GI-NEXT:    lsl w14, w14, #83736; CHECK-GI-NEXT:    lsl w13, w13, #83737; CHECK-GI-NEXT:    sbfx w16, w10, #8, #83738; CHECK-GI-NEXT:    lsl w15, w15, #83739; CHECK-GI-NEXT:    mov v20.h[5], w123740; CHECK-GI-NEXT:    ldr w12, [sp, #728]3741; CHECK-GI-NEXT:    sbfx w14, w14, #8, #83742; CHECK-GI-NEXT:    sbfx w13, w13, #8, #83743; CHECK-GI-NEXT:    mov v19.h[7], w163744; CHECK-GI-NEXT:    ldr w9, [sp, #472]3745; CHECK-GI-NEXT:    lsl w12, w12, #83746; CHECK-GI-NEXT:    mov v18.h[6], w143747; CHECK-GI-NEXT:    sbfx w14, w15, #8, #83748; CHECK-GI-NEXT:    mov v21.h[5], w133749; CHECK-GI-NEXT:    ldr w15, [sp, #672]3750; CHECK-GI-NEXT:    ldr w11, [sp, #536]3751; CHECK-GI-NEXT:    ldr w13, [sp, #736]3752; CHECK-GI-NEXT:    sbfx w12, w12, #8, #83753; CHECK-GI-NEXT:    lsl w9, w9, #83754; CHECK-GI-NEXT:    mov v20.h[6], w143755; CHECK-GI-NEXT:    lsl w15, w15, #83756; CHECK-GI-NEXT:    lsl w11, w11, #83757; CHECK-GI-NEXT:    mul v19.8h, v16.8h, v19.8h3758; CHECK-GI-NEXT:    lsl w13, w13, #83759; CHECK-GI-NEXT:    sbfx w9, w9, #8, #83760; CHECK-GI-NEXT:    mov v21.h[6], w123761; CHECK-GI-NEXT:    sbfx w15, w15, #8, #83762; CHECK-GI-NEXT:    smov w14, v7.h[2]3763; CHECK-GI-NEXT:    sbfx w11, w11, #8, #83764; CHECK-GI-NEXT:    sbfx w13, w13, #8, #83765; CHECK-GI-NEXT:    smov w12, v7.h[4]3766; CHECK-GI-NEXT:    mov v17.h[7], w93767; CHECK-GI-NEXT:    mov v20.h[7], w153768; CHECK-GI-NEXT:    smov w9, v7.h[5]3769; CHECK-GI-NEXT:    mov v18.h[7], w113770; CHECK-GI-NEXT:    smov w11, v19.h[4]3771; CHECK-GI-NEXT:    ldr w15, [sp, #744]3772; CHECK-GI-NEXT:    mov v21.h[7], w133773; CHECK-GI-NEXT:    mov v6.s[2], w143774; CHECK-GI-NEXT:    smov w14, v19.h[0]3775; CHECK-GI-NEXT:    fmov s16, w123776; CHECK-GI-NEXT:    smov w13, v19.h[5]3777; CHECK-GI-NEXT:    smov w12, v19.h[1]3778; CHECK-GI-NEXT:    mul v20.8h, v17.8h, v20.8h3779; CHECK-GI-NEXT:    ldr w10, [sp, #544]3780; CHECK-GI-NEXT:    add v3.4s, v4.4s, v5.4s3781; CHECK-GI-NEXT:    mul v22.8h, v18.8h, v21.8h3782; CHECK-GI-NEXT:    fmov s18, w113783; CHECK-GI-NEXT:    mov v16.s[1], w93784; CHECK-GI-NEXT:    fmov s17, w143785; CHECK-GI-NEXT:    smov w14, v7.h[6]3786; CHECK-GI-NEXT:    smov w11, v19.h[2]3787; CHECK-GI-NEXT:    smov w9, v7.h[3]3788; CHECK-GI-NEXT:    sxtb w10, w103789; CHECK-GI-NEXT:    fmov s21, w83790; CHECK-GI-NEXT:    mov v18.s[1], w133791; CHECK-GI-NEXT:    sxtb w13, w153792; CHECK-GI-NEXT:    smov w15, v20.h[0]3793; CHECK-GI-NEXT:    mov v17.s[1], w123794; CHECK-GI-NEXT:    smov w8, v7.h[7]3795; CHECK-GI-NEXT:    smov w12, v19.h[6]3796; CHECK-GI-NEXT:    mov v16.s[2], w143797; CHECK-GI-NEXT:    smov w14, v20.h[1]3798; CHECK-GI-NEXT:    mul w10, w10, w133799; CHECK-GI-NEXT:    smov w13, v20.h[4]3800; CHECK-GI-NEXT:    smov w16, v20.h[5]3801; CHECK-GI-NEXT:    mov v21.s[1], wzr3802; CHECK-GI-NEXT:    fmov s7, w153803; CHECK-GI-NEXT:    smov w15, v20.h[2]3804; CHECK-GI-NEXT:    mov v6.s[3], w93805; CHECK-GI-NEXT:    mov v17.s[2], w113806; CHECK-GI-NEXT:    smov w11, v22.h[0]3807; CHECK-GI-NEXT:    sxth w10, w103808; CHECK-GI-NEXT:    mov v18.s[2], w123809; CHECK-GI-NEXT:    smov w12, v22.h[1]3810; CHECK-GI-NEXT:    mov v16.s[3], w83811; CHECK-GI-NEXT:    mov v7.s[1], w143812; CHECK-GI-NEXT:    smov w14, v22.h[4]3813; CHECK-GI-NEXT:    fmov s23, w133814; CHECK-GI-NEXT:    smov w13, v22.h[5]3815; CHECK-GI-NEXT:    fmov s26, w103816; CHECK-GI-NEXT:    smov w10, v19.h[7]3817; CHECK-GI-NEXT:    fmov s24, w113818; CHECK-GI-NEXT:    smov w11, v20.h[6]3819; CHECK-GI-NEXT:    mov v21.s[2], wzr3820; CHECK-GI-NEXT:    mov v23.s[1], w163821; CHECK-GI-NEXT:    add v4.4s, v6.4s, v16.4s3822; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s3823; CHECK-GI-NEXT:    fmov s25, w143824; CHECK-GI-NEXT:    smov w14, v22.h[2]3825; CHECK-GI-NEXT:    mov v26.s[1], wzr3826; CHECK-GI-NEXT:    mov v24.s[1], w123827; CHECK-GI-NEXT:    smov w12, v19.h[3]3828; CHECK-GI-NEXT:    mov v7.s[2], w153829; CHECK-GI-NEXT:    smov w15, v20.h[3]3830; CHECK-GI-NEXT:    mov v18.s[3], w103831; CHECK-GI-NEXT:    mov v21.s[3], wzr3832; CHECK-GI-NEXT:    mov v25.s[1], w133833; CHECK-GI-NEXT:    smov w13, v22.h[6]3834; CHECK-GI-NEXT:    mov v23.s[2], w113835; CHECK-GI-NEXT:    smov w11, v20.h[7]3836; CHECK-GI-NEXT:    mov v26.s[2], wzr3837; CHECK-GI-NEXT:    mov v24.s[2], w143838; CHECK-GI-NEXT:    smov w14, v22.h[3]3839; CHECK-GI-NEXT:    mov v17.s[3], w123840; CHECK-GI-NEXT:    mov v7.s[3], w153841; CHECK-GI-NEXT:    add v1.4s, v21.4s, v1.4s3842; CHECK-GI-NEXT:    mov v25.s[2], w133843; CHECK-GI-NEXT:    smov w13, v22.h[7]3844; CHECK-GI-NEXT:    mov v23.s[3], w113845; CHECK-GI-NEXT:    mov v26.s[3], wzr3846; CHECK-GI-NEXT:    mov v24.s[3], w143847; CHECK-GI-NEXT:    add v5.4s, v17.4s, v18.4s3848; CHECK-GI-NEXT:    add v1.4s, v4.4s, v1.4s3849; CHECK-GI-NEXT:    mov v25.s[3], w133850; CHECK-GI-NEXT:    add v6.4s, v7.4s, v23.4s3851; CHECK-GI-NEXT:    add v0.4s, v26.4s, v0.4s3852; CHECK-GI-NEXT:    add v1.4s, v2.4s, v1.4s3853; CHECK-GI-NEXT:    add v7.4s, v24.4s, v25.4s3854; CHECK-GI-NEXT:    add v3.4s, v5.4s, v6.4s3855; CHECK-GI-NEXT:    addv s1, v1.4s3856; CHECK-GI-NEXT:    add v0.4s, v7.4s, v0.4s3857; CHECK-GI-NEXT:    fmov w8, s13858; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s3859; CHECK-GI-NEXT:    addv s0, v0.4s3860; CHECK-GI-NEXT:    fmov w9, s03861; CHECK-GI-NEXT:    add w0, w8, w93862; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload3863; CHECK-GI-NEXT:    ret3864entry:3865  %az = sext <25 x i8> %a to <25 x i32>3866  %bz = sext <25 x i8> %b to <25 x i32>3867  %m1 = mul nuw nsw <25 x i32> %az, %bz3868  %r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m1)3869  %cz = sext <25 x i8> %c to <25 x i32>3870  %dz = sext <25 x i8> %d to <25 x i32>3871  %m2 = mul nuw nsw <25 x i32> %cz, %dz3872  %r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m2)3873  %x = add i32 %r1, %r23874  ret i32 %x3875}3876 3877define i32 @test_sdot_v25i8_double_nomla(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) {3878; CHECK-SD-LABEL: test_sdot_v25i8_double_nomla:3879; CHECK-SD:       // %bb.0: // %entry3880; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill3881; CHECK-SD-NEXT:    .cfi_def_cfa_offset 163882; CHECK-SD-NEXT:    .cfi_offset w29, -163883; CHECK-SD-NEXT:    fmov s0, w03884; CHECK-SD-NEXT:    ldr b1, [sp, #80]3885; CHECK-SD-NEXT:    add x10, sp, #883886; CHECK-SD-NEXT:    ldr b2, [sp, #16]3887; CHECK-SD-NEXT:    add x9, sp, #963888; CHECK-SD-NEXT:    ldr b3, [sp, #480]3889; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x10]3890; CHECK-SD-NEXT:    add x10, sp, #243891; CHECK-SD-NEXT:    ldr b4, [sp, #352]3892; CHECK-SD-NEXT:    mov v0.b[1], w13893; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x10]3894; CHECK-SD-NEXT:    add x11, sp, #4883895; CHECK-SD-NEXT:    add x10, sp, #3603896; CHECK-SD-NEXT:    ldr b5, [sp, #416]3897; CHECK-SD-NEXT:    add x8, sp, #1043898; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x9]3899; CHECK-SD-NEXT:    add x9, sp, #323900; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x11]3901; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x9]3902; CHECK-SD-NEXT:    add x11, sp, #4243903; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x10]3904; CHECK-SD-NEXT:    mov v0.b[2], w23905; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x11]3906; CHECK-SD-NEXT:    add x9, sp, #3683907; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]3908; CHECK-SD-NEXT:    add x8, sp, #403909; CHECK-SD-NEXT:    add x12, sp, #4963910; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x8]3911; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x9]3912; CHECK-SD-NEXT:    add x8, sp, #4323913; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x12]3914; CHECK-SD-NEXT:    add x13, sp, #483915; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x8]3916; CHECK-SD-NEXT:    mov v0.b[3], w33917; CHECK-SD-NEXT:    add x10, sp, #1123918; CHECK-SD-NEXT:    add x8, sp, #5043919; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x13]3920; CHECK-SD-NEXT:    add x13, sp, #3763921; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x10]3922; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x13]3923; CHECK-SD-NEXT:    add x13, sp, #4403924; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x8]3925; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x13]3926; CHECK-SD-NEXT:    add x11, sp, #1203927; CHECK-SD-NEXT:    add x8, sp, #563928; CHECK-SD-NEXT:    mov v0.b[4], w43929; CHECK-SD-NEXT:    add x13, sp, #5123930; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x11]3931; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x8]3932; CHECK-SD-NEXT:    add x8, sp, #3843933; CHECK-SD-NEXT:    add x11, sp, #4483934; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x13]3935; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]3936; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x11]3937; CHECK-SD-NEXT:    add x12, sp, #1283938; CHECK-SD-NEXT:    add x10, sp, #643939; CHECK-SD-NEXT:    add x8, sp, #5203940; CHECK-SD-NEXT:    mov v0.b[5], w53941; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x12]3942; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x10]3943; CHECK-SD-NEXT:    add x10, sp, #3923944; CHECK-SD-NEXT:    add x11, sp, #4563945; CHECK-SD-NEXT:    ldr b6, [sp, #144]3946; CHECK-SD-NEXT:    ldr b7, [sp, #544]3947; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x8]3948; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x10]3949; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x11]3950; CHECK-SD-NEXT:    add x9, sp, #1363951; CHECK-SD-NEXT:    sshll v6.8h, v6.8b, #03952; CHECK-SD-NEXT:    mov v0.b[6], w63953; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x9]3954; CHECK-SD-NEXT:    add x8, sp, #5283955; CHECK-SD-NEXT:    add x9, sp, #4003956; CHECK-SD-NEXT:    add x10, sp, #4643957; CHECK-SD-NEXT:    sshll v7.8h, v7.8b, #03958; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x8]3959; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x9]3960; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x10]3961; CHECK-SD-NEXT:    movi v16.2d, #00000000000000003962; CHECK-SD-NEXT:    movi v17.2d, #00000000000000003963; CHECK-SD-NEXT:    add x14, sp, #723964; CHECK-SD-NEXT:    mov v0.b[7], w73965; CHECK-SD-NEXT:    sshll v6.4s, v6.4h, #03966; CHECK-SD-NEXT:    add x8, sp, #5363967; CHECK-SD-NEXT:    add x9, sp, #4083968; CHECK-SD-NEXT:    add x10, sp, #4723969; CHECK-SD-NEXT:    sshll v7.4s, v7.4h, #03970; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x14]3971; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x8]3972; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x9]3973; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x10]3974; CHECK-SD-NEXT:    mov v16.s[0], v6.s[0]3975; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #03976; CHECK-SD-NEXT:    mov v17.s[0], v7.s[0]3977; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #03978; CHECK-SD-NEXT:    sshll v2.8h, v2.8b, #03979; CHECK-SD-NEXT:    sshll v3.8h, v3.8b, #03980; CHECK-SD-NEXT:    sshll v4.8h, v4.8b, #03981; CHECK-SD-NEXT:    sshll v5.8h, v5.8b, #03982; CHECK-SD-NEXT:    saddl v7.4s, v0.4h, v1.4h3983; CHECK-SD-NEXT:    saddl2 v0.4s, v0.8h, v1.8h3984; CHECK-SD-NEXT:    saddw v6.4s, v16.4s, v2.4h3985; CHECK-SD-NEXT:    saddl v1.4s, v4.4h, v3.4h3986; CHECK-SD-NEXT:    saddl2 v3.4s, v4.8h, v3.8h3987; CHECK-SD-NEXT:    saddw v4.4s, v17.4s, v5.4h3988; CHECK-SD-NEXT:    saddw2 v0.4s, v0.4s, v2.8h3989; CHECK-SD-NEXT:    add v6.4s, v7.4s, v6.4s3990; CHECK-SD-NEXT:    saddw2 v2.4s, v3.4s, v5.8h3991; CHECK-SD-NEXT:    add v1.4s, v1.4s, v4.4s3992; CHECK-SD-NEXT:    add v0.4s, v6.4s, v0.4s3993; CHECK-SD-NEXT:    add v1.4s, v1.4s, v2.4s3994; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s3995; CHECK-SD-NEXT:    addv s0, v0.4s3996; CHECK-SD-NEXT:    fmov w0, s03997; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload3998; CHECK-SD-NEXT:    ret3999;4000; CHECK-GI-LABEL: test_sdot_v25i8_double_nomla:4001; CHECK-GI:       // %bb.0: // %entry4002; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill4003; CHECK-GI-NEXT:    .cfi_def_cfa_offset 164004; CHECK-GI-NEXT:    .cfi_offset w29, -164005; CHECK-GI-NEXT:    sxtb w8, w04006; CHECK-GI-NEXT:    sxtb w9, w14007; CHECK-GI-NEXT:    ldr w10, [sp, #48]4008; CHECK-GI-NEXT:    sxtb w11, w64009; CHECK-GI-NEXT:    sxtb w12, w34010; CHECK-GI-NEXT:    sxtb w13, w74011; CHECK-GI-NEXT:    fmov s2, w84012; CHECK-GI-NEXT:    sxtb w8, w44013; CHECK-GI-NEXT:    sxtb w10, w104014; CHECK-GI-NEXT:    ldr w14, [sp, #448]4015; CHECK-GI-NEXT:    movi d1, #00000000000000004016; CHECK-GI-NEXT:    movi d0, #00000000000000004017; CHECK-GI-NEXT:    fmov s3, w84018; CHECK-GI-NEXT:    sxtb w8, w24019; CHECK-GI-NEXT:    fmov s5, w104020; CHECK-GI-NEXT:    mov v2.s[1], w94021; CHECK-GI-NEXT:    sxtb w9, w54022; CHECK-GI-NEXT:    ldr w10, [sp, #80]4023; CHECK-GI-NEXT:    mov v3.s[1], w94024; CHECK-GI-NEXT:    ldr w9, [sp, #16]4025; CHECK-GI-NEXT:    sxtb w10, w104026; CHECK-GI-NEXT:    mov v1.s[1], wzr4027; CHECK-GI-NEXT:    mov v0.s[1], wzr4028; CHECK-GI-NEXT:    mov v2.s[2], w84029; CHECK-GI-NEXT:    sxtb w9, w94030; CHECK-GI-NEXT:    ldr w8, [sp, #24]4031; CHECK-GI-NEXT:    fmov s6, w104032; CHECK-GI-NEXT:    ldr w10, [sp, #64]4033; CHECK-GI-NEXT:    mov v3.s[2], w114034; CHECK-GI-NEXT:    fmov s4, w94035; CHECK-GI-NEXT:    sxtb w8, w84036; CHECK-GI-NEXT:    ldr w9, [sp, #56]4037; CHECK-GI-NEXT:    ldr w11, [sp, #32]4038; CHECK-GI-NEXT:    sxtb w10, w104039; CHECK-GI-NEXT:    mov v2.s[3], w124040; CHECK-GI-NEXT:    ldr w12, [sp, #88]4041; CHECK-GI-NEXT:    mov v1.s[2], wzr4042; CHECK-GI-NEXT:    mov v4.s[1], w84043; CHECK-GI-NEXT:    ldr w8, [sp, #120]4044; CHECK-GI-NEXT:    sxtb w9, w94045; CHECK-GI-NEXT:    mov v3.s[3], w134046; CHECK-GI-NEXT:    ldr w13, [sp, #112]4047; CHECK-GI-NEXT:    sxtb w12, w124048; CHECK-GI-NEXT:    sxtb w8, w84049; CHECK-GI-NEXT:    sxtb w11, w114050; CHECK-GI-NEXT:    mov v5.s[1], w94051; CHECK-GI-NEXT:    sxtb w13, w134052; CHECK-GI-NEXT:    mov v6.s[1], w124053; CHECK-GI-NEXT:    ldr w12, [sp, #128]4054; CHECK-GI-NEXT:    mov v4.s[2], w114055; CHECK-GI-NEXT:    ldr w11, [sp, #104]4056; CHECK-GI-NEXT:    ldr w9, [sp, #40]4057; CHECK-GI-NEXT:    fmov s7, w134058; CHECK-GI-NEXT:    ldr w13, [sp, #96]4059; CHECK-GI-NEXT:    sxtb w12, w124060; CHECK-GI-NEXT:    sxtb w11, w114061; CHECK-GI-NEXT:    mov v5.s[2], w104062; CHECK-GI-NEXT:    ldr w10, [sp, #136]4063; CHECK-GI-NEXT:    sxtb w13, w134064; CHECK-GI-NEXT:    sxtb w9, w94065; CHECK-GI-NEXT:    mov v0.s[2], wzr4066; CHECK-GI-NEXT:    mov v7.s[1], w84067; CHECK-GI-NEXT:    sxtb w10, w104068; CHECK-GI-NEXT:    ldr w8, [sp, #72]4069; CHECK-GI-NEXT:    mov v6.s[2], w134070; CHECK-GI-NEXT:    ldr w13, [sp, #144]4071; CHECK-GI-NEXT:    mov v4.s[3], w94072; CHECK-GI-NEXT:    ldr w9, [sp, #360]4073; CHECK-GI-NEXT:    sxtb w8, w84074; CHECK-GI-NEXT:    mov v1.s[3], wzr4075; CHECK-GI-NEXT:    sxtb w13, w134076; CHECK-GI-NEXT:    mov v0.s[3], wzr4077; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s4078; CHECK-GI-NEXT:    mov v7.s[2], w124079; CHECK-GI-NEXT:    ldr w12, [sp, #352]4080; CHECK-GI-NEXT:    sxtb w9, w94081; CHECK-GI-NEXT:    mov v6.s[3], w114082; CHECK-GI-NEXT:    ldr w11, [sp, #384]4083; CHECK-GI-NEXT:    fmov s16, w134084; CHECK-GI-NEXT:    sxtb w12, w124085; CHECK-GI-NEXT:    ldr w13, [sp, #416]4086; CHECK-GI-NEXT:    mov v5.s[3], w84087; CHECK-GI-NEXT:    sxtb w11, w114088; CHECK-GI-NEXT:    ldr w8, [sp, #368]4089; CHECK-GI-NEXT:    mov v7.s[3], w104090; CHECK-GI-NEXT:    ldr w10, [sp, #392]4091; CHECK-GI-NEXT:    fmov s17, w124092; CHECK-GI-NEXT:    fmov s18, w114093; CHECK-GI-NEXT:    ldr w12, [sp, #424]4094; CHECK-GI-NEXT:    ldr w11, [sp, #456]4095; CHECK-GI-NEXT:    sxtb w10, w104096; CHECK-GI-NEXT:    sxtb w13, w134097; CHECK-GI-NEXT:    sxtb w8, w84098; CHECK-GI-NEXT:    sxtb w11, w114099; CHECK-GI-NEXT:    mov v17.s[1], w94100; CHECK-GI-NEXT:    mov v16.s[1], wzr4101; CHECK-GI-NEXT:    mov v18.s[1], w104102; CHECK-GI-NEXT:    sxtb w10, w124103; CHECK-GI-NEXT:    sxtb w12, w144104; CHECK-GI-NEXT:    fmov s19, w134105; CHECK-GI-NEXT:    ldr w13, [sp, #400]4106; CHECK-GI-NEXT:    ldr w9, [sp, #376]4107; CHECK-GI-NEXT:    fmov s20, w124108; CHECK-GI-NEXT:    ldr w12, [sp, #464]4109; CHECK-GI-NEXT:    add v3.4s, v4.4s, v5.4s4110; CHECK-GI-NEXT:    sxtb w13, w134111; CHECK-GI-NEXT:    mov v17.s[2], w84112; CHECK-GI-NEXT:    ldr w8, [sp, #488]4113; CHECK-GI-NEXT:    mov v19.s[1], w104114; CHECK-GI-NEXT:    ldr w10, [sp, #432]4115; CHECK-GI-NEXT:    sxtb w12, w124116; CHECK-GI-NEXT:    mov v20.s[1], w114117; CHECK-GI-NEXT:    ldr w11, [sp, #480]4118; CHECK-GI-NEXT:    mov v18.s[2], w134119; CHECK-GI-NEXT:    ldr w13, [sp, #512]4120; CHECK-GI-NEXT:    sxtb w10, w104121; CHECK-GI-NEXT:    sxtb w8, w84122; CHECK-GI-NEXT:    sxtb w11, w114123; CHECK-GI-NEXT:    mov v16.s[2], wzr4124; CHECK-GI-NEXT:    sxtb w9, w94125; CHECK-GI-NEXT:    sxtb w13, w134126; CHECK-GI-NEXT:    mov v19.s[2], w104127; CHECK-GI-NEXT:    ldr w10, [sp, #520]4128; CHECK-GI-NEXT:    fmov s21, w114129; CHECK-GI-NEXT:    ldr w11, [sp, #544]4130; CHECK-GI-NEXT:    mov v20.s[2], w124131; CHECK-GI-NEXT:    fmov s22, w134132; CHECK-GI-NEXT:    sxtb w10, w104133; CHECK-GI-NEXT:    ldr w13, [sp, #528]4134; CHECK-GI-NEXT:    sxtb w11, w114135; CHECK-GI-NEXT:    ldr w12, [sp, #408]4136; CHECK-GI-NEXT:    mov v17.s[3], w94137; CHECK-GI-NEXT:    mov v21.s[1], w84138; CHECK-GI-NEXT:    ldr w8, [sp, #496]4139; CHECK-GI-NEXT:    sxtb w13, w134140; CHECK-GI-NEXT:    fmov s23, w114141; CHECK-GI-NEXT:    mov v22.s[1], w104142; CHECK-GI-NEXT:    ldr w10, [sp, #440]4143; CHECK-GI-NEXT:    sxtb w8, w84144; CHECK-GI-NEXT:    ldr w11, [sp, #472]4145; CHECK-GI-NEXT:    sxtb w12, w124146; CHECK-GI-NEXT:    sxtb w10, w104147; CHECK-GI-NEXT:    mov v16.s[3], wzr4148; CHECK-GI-NEXT:    add v4.4s, v6.4s, v7.4s4149; CHECK-GI-NEXT:    mov v23.s[1], wzr4150; CHECK-GI-NEXT:    mov v21.s[2], w84151; CHECK-GI-NEXT:    ldr w8, [sp, #504]4152; CHECK-GI-NEXT:    mov v22.s[2], w134153; CHECK-GI-NEXT:    ldr w13, [sp, #536]4154; CHECK-GI-NEXT:    sxtb w11, w114155; CHECK-GI-NEXT:    sxtb w8, w84156; CHECK-GI-NEXT:    mov v18.s[3], w124157; CHECK-GI-NEXT:    mov v19.s[3], w104158; CHECK-GI-NEXT:    sxtb w9, w134159; CHECK-GI-NEXT:    mov v20.s[3], w114160; CHECK-GI-NEXT:    add v1.4s, v16.4s, v1.4s4161; CHECK-GI-NEXT:    mov v23.s[2], wzr4162; CHECK-GI-NEXT:    mov v21.s[3], w84163; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s4164; CHECK-GI-NEXT:    mov v22.s[3], w94165; CHECK-GI-NEXT:    add v5.4s, v17.4s, v18.4s4166; CHECK-GI-NEXT:    add v1.4s, v4.4s, v1.4s4167; CHECK-GI-NEXT:    add v6.4s, v19.4s, v20.4s4168; CHECK-GI-NEXT:    mov v23.s[3], wzr4169; CHECK-GI-NEXT:    add v7.4s, v21.4s, v22.4s4170; CHECK-GI-NEXT:    add v1.4s, v2.4s, v1.4s4171; CHECK-GI-NEXT:    add v3.4s, v5.4s, v6.4s4172; CHECK-GI-NEXT:    add v0.4s, v23.4s, v0.4s4173; CHECK-GI-NEXT:    addv s1, v1.4s4174; CHECK-GI-NEXT:    add v0.4s, v7.4s, v0.4s4175; CHECK-GI-NEXT:    fmov w8, s14176; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s4177; CHECK-GI-NEXT:    addv s0, v0.4s4178; CHECK-GI-NEXT:    fmov w9, s04179; CHECK-GI-NEXT:    add w0, w8, w94180; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload4181; CHECK-GI-NEXT:    ret4182entry:4183  %az = sext <25 x i8> %a to <25 x i32>4184  %r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %az)4185  %cz = sext <25 x i8> %c to <25 x i32>4186  %r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %cz)4187  %x = add i32 %r1, %r24188  ret i32 %x4189}4190 4191define i32 @test_udot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {4192; CHECK-SD-LABEL: test_udot_v32i8:4193; CHECK-SD:       // %bb.0: // %entry4194; CHECK-SD-NEXT:    movi v0.2d, #00000000000000004195; CHECK-SD-NEXT:    ldp q1, q3, [x0]4196; CHECK-SD-NEXT:    ldp q2, q4, [x1]4197; CHECK-SD-NEXT:    udot v0.4s, v4.16b, v3.16b4198; CHECK-SD-NEXT:    udot v0.4s, v2.16b, v1.16b4199; CHECK-SD-NEXT:    addv s0, v0.4s4200; CHECK-SD-NEXT:    fmov w8, s04201; CHECK-SD-NEXT:    add w0, w8, w24202; CHECK-SD-NEXT:    ret4203;4204; CHECK-GI-LABEL: test_udot_v32i8:4205; CHECK-GI:       // %bb.0: // %entry4206; CHECK-GI-NEXT:    movi v0.2d, #00000000000000004207; CHECK-GI-NEXT:    movi v1.2d, #00000000000000004208; CHECK-GI-NEXT:    ldp q2, q3, [x0]4209; CHECK-GI-NEXT:    ldp q4, q5, [x1]4210; CHECK-GI-NEXT:    udot v1.4s, v4.16b, v2.16b4211; CHECK-GI-NEXT:    udot v0.4s, v5.16b, v3.16b4212; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s4213; CHECK-GI-NEXT:    addv s0, v0.4s4214; CHECK-GI-NEXT:    fmov w8, s04215; CHECK-GI-NEXT:    add w0, w8, w24216; CHECK-GI-NEXT:    ret4217entry:4218  %0 = load <32 x i8>, ptr %a4219  %1 = zext <32 x i8> %0 to <32 x i32>4220  %2 = load <32 x i8>, ptr %b4221  %3 = zext <32 x i8> %2 to <32 x i32>4222  %4 = mul nuw nsw <32 x i32> %3, %14223  %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4)4224  %op.extra = add i32 %5, %sum4225  ret i32 %op.extra4226}4227 4228define i32 @test_udot_v32i8_nomla(ptr nocapture readonly %a1) {4229; CHECK-SD-LABEL: test_udot_v32i8_nomla:4230; CHECK-SD:       // %bb.0: // %entry4231; CHECK-SD-NEXT:    movi v0.16b, #14232; CHECK-SD-NEXT:    movi v1.2d, #00000000000000004233; CHECK-SD-NEXT:    ldp q2, q3, [x0]4234; CHECK-SD-NEXT:    udot v1.4s, v3.16b, v0.16b4235; CHECK-SD-NEXT:    udot v1.4s, v2.16b, v0.16b4236; CHECK-SD-NEXT:    addv s0, v1.4s4237; CHECK-SD-NEXT:    fmov w0, s04238; CHECK-SD-NEXT:    ret4239;4240; CHECK-GI-LABEL: test_udot_v32i8_nomla:4241; CHECK-GI:       // %bb.0: // %entry4242; CHECK-GI-NEXT:    movi v0.16b, #14243; CHECK-GI-NEXT:    movi v1.2d, #00000000000000004244; CHECK-GI-NEXT:    movi v2.2d, #00000000000000004245; CHECK-GI-NEXT:    ldp q3, q4, [x0]4246; CHECK-GI-NEXT:    udot v2.4s, v3.16b, v0.16b4247; CHECK-GI-NEXT:    udot v1.4s, v4.16b, v0.16b4248; CHECK-GI-NEXT:    add v0.4s, v2.4s, v1.4s4249; CHECK-GI-NEXT:    addv s0, v0.4s4250; CHECK-GI-NEXT:    fmov w0, s04251; CHECK-GI-NEXT:    ret4252entry:4253  %0 = load <32 x i8>, ptr %a14254  %1 = zext <32 x i8> %0 to <32 x i32>4255  %2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %1)4256  ret i32 %24257}4258define i32 @test_sdot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {4259; CHECK-SD-LABEL: test_sdot_v32i8:4260; CHECK-SD:       // %bb.0: // %entry4261; CHECK-SD-NEXT:    movi v0.2d, #00000000000000004262; CHECK-SD-NEXT:    ldp q1, q3, [x0]4263; CHECK-SD-NEXT:    ldp q2, q4, [x1]4264; CHECK-SD-NEXT:    sdot v0.4s, v4.16b, v3.16b4265; CHECK-SD-NEXT:    sdot v0.4s, v2.16b, v1.16b4266; CHECK-SD-NEXT:    addv s0, v0.4s4267; CHECK-SD-NEXT:    fmov w8, s04268; CHECK-SD-NEXT:    add w0, w8, w24269; CHECK-SD-NEXT:    ret4270;4271; CHECK-GI-LABEL: test_sdot_v32i8:4272; CHECK-GI:       // %bb.0: // %entry4273; CHECK-GI-NEXT:    movi v0.2d, #00000000000000004274; CHECK-GI-NEXT:    movi v1.2d, #00000000000000004275; CHECK-GI-NEXT:    ldp q2, q3, [x0]4276; CHECK-GI-NEXT:    ldp q4, q5, [x1]4277; CHECK-GI-NEXT:    sdot v1.4s, v4.16b, v2.16b4278; CHECK-GI-NEXT:    sdot v0.4s, v5.16b, v3.16b4279; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s4280; CHECK-GI-NEXT:    addv s0, v0.4s4281; CHECK-GI-NEXT:    fmov w8, s04282; CHECK-GI-NEXT:    add w0, w8, w24283; CHECK-GI-NEXT:    ret4284entry:4285  %0 = load <32 x i8>, ptr %a4286  %1 = sext <32 x i8> %0 to <32 x i32>4287  %2 = load <32 x i8>, ptr %b4288  %3 = sext <32 x i8> %2 to <32 x i32>4289  %4 = mul nsw <32 x i32> %3, %14290  %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4)4291  %op.extra = add nsw i32 %5, %sum4292  ret i32 %op.extra4293}4294 4295define i32 @test_sdot_v32i8_double(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {4296; CHECK-SD-LABEL: test_sdot_v32i8_double:4297; CHECK-SD:       // %bb.0: // %entry4298; CHECK-SD-NEXT:    movi v16.2d, #00000000000000004299; CHECK-SD-NEXT:    movi v17.2d, #00000000000000004300; CHECK-SD-NEXT:    sdot v17.4s, v1.16b, v3.16b4301; CHECK-SD-NEXT:    sdot v16.4s, v5.16b, v7.16b4302; CHECK-SD-NEXT:    sdot v17.4s, v0.16b, v2.16b4303; CHECK-SD-NEXT:    sdot v16.4s, v4.16b, v6.16b4304; CHECK-SD-NEXT:    add v0.4s, v17.4s, v16.4s4305; CHECK-SD-NEXT:    addv s0, v0.4s4306; CHECK-SD-NEXT:    fmov w0, s04307; CHECK-SD-NEXT:    ret4308;4309; CHECK-GI-LABEL: test_sdot_v32i8_double:4310; CHECK-GI:       // %bb.0: // %entry4311; CHECK-GI-NEXT:    movi v16.2d, #00000000000000004312; CHECK-GI-NEXT:    movi v17.2d, #00000000000000004313; CHECK-GI-NEXT:    movi v18.2d, #00000000000000004314; CHECK-GI-NEXT:    movi v19.2d, #00000000000000004315; CHECK-GI-NEXT:    sdot v16.4s, v0.16b, v2.16b4316; CHECK-GI-NEXT:    sdot v18.4s, v1.16b, v3.16b4317; CHECK-GI-NEXT:    sdot v17.4s, v5.16b, v7.16b4318; CHECK-GI-NEXT:    sdot v19.4s, v4.16b, v6.16b4319; CHECK-GI-NEXT:    add v0.4s, v16.4s, v18.4s4320; CHECK-GI-NEXT:    add v1.4s, v19.4s, v17.4s4321; CHECK-GI-NEXT:    addv s0, v0.4s4322; CHECK-GI-NEXT:    addv s1, v1.4s4323; CHECK-GI-NEXT:    fmov w8, s04324; CHECK-GI-NEXT:    fmov w9, s14325; CHECK-GI-NEXT:    add w0, w8, w94326; CHECK-GI-NEXT:    ret4327entry:4328  %az = sext <32 x i8> %a to <32 x i32>4329  %bz = sext <32 x i8> %b to <32 x i32>4330  %m1 = mul nuw nsw <32 x i32> %az, %bz4331  %r1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m1)4332  %cz = sext <32 x i8> %c to <32 x i32>4333  %dz = sext <32 x i8> %d to <32 x i32>4334  %m2 = mul nuw nsw <32 x i32> %cz, %dz4335  %r2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m2)4336  %x = add i32 %r1, %r24337  ret i32 %x4338}4339 4340define i32 @test_sdot_v32i8_double_nomla(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {4341; CHECK-SD-LABEL: test_sdot_v32i8_double_nomla:4342; CHECK-SD:       // %bb.0: // %entry4343; CHECK-SD-NEXT:    movi v2.16b, #14344; CHECK-SD-NEXT:    movi v3.2d, #00000000000000004345; CHECK-SD-NEXT:    movi v6.2d, #00000000000000004346; CHECK-SD-NEXT:    sdot v6.4s, v1.16b, v2.16b4347; CHECK-SD-NEXT:    sdot v3.4s, v5.16b, v2.16b4348; CHECK-SD-NEXT:    sdot v6.4s, v0.16b, v2.16b4349; CHECK-SD-NEXT:    sdot v3.4s, v4.16b, v2.16b4350; CHECK-SD-NEXT:    add v0.4s, v6.4s, v3.4s4351; CHECK-SD-NEXT:    addv s0, v0.4s4352; CHECK-SD-NEXT:    fmov w0, s04353; CHECK-SD-NEXT:    ret4354;4355; CHECK-GI-LABEL: test_sdot_v32i8_double_nomla:4356; CHECK-GI:       // %bb.0: // %entry4357; CHECK-GI-NEXT:    movi v2.16b, #14358; CHECK-GI-NEXT:    movi v3.2d, #00000000000000004359; CHECK-GI-NEXT:    movi v6.2d, #00000000000000004360; CHECK-GI-NEXT:    movi v7.2d, #00000000000000004361; CHECK-GI-NEXT:    movi v16.2d, #00000000000000004362; CHECK-GI-NEXT:    sdot v3.4s, v0.16b, v2.16b4363; CHECK-GI-NEXT:    sdot v6.4s, v5.16b, v2.16b4364; CHECK-GI-NEXT:    sdot v7.4s, v1.16b, v2.16b4365; CHECK-GI-NEXT:    sdot v16.4s, v4.16b, v2.16b4366; CHECK-GI-NEXT:    add v0.4s, v3.4s, v7.4s4367; CHECK-GI-NEXT:    add v1.4s, v16.4s, v6.4s4368; CHECK-GI-NEXT:    addv s0, v0.4s4369; CHECK-GI-NEXT:    addv s1, v1.4s4370; CHECK-GI-NEXT:    fmov w8, s04371; CHECK-GI-NEXT:    fmov w9, s14372; CHECK-GI-NEXT:    add w0, w8, w94373; CHECK-GI-NEXT:    ret4374entry:4375  %az = sext <32 x i8> %a to <32 x i32>4376  %r1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %az)4377  %cz = sext <32 x i8> %c to <32 x i32>4378  %r2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %cz)4379  %x = add i32 %r1, %r24380  ret i32 %x4381}4382 4383define i32 @test_usdot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {4384; CHECK-SD-LABEL: test_usdot_v32i8:4385; CHECK-SD:       // %bb.0: // %entry4386; CHECK-SD-NEXT:    movi v0.2d, #00000000000000004387; CHECK-SD-NEXT:    movi v1.2d, #00000000000000004388; CHECK-SD-NEXT:    ldp q2, q3, [x0]4389; CHECK-SD-NEXT:    ldp q4, q5, [x1]4390; CHECK-SD-NEXT:    usdot v1.4s, v3.16b, v5.16b4391; CHECK-SD-NEXT:    usdot v0.4s, v2.16b, v4.16b4392; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s4393; CHECK-SD-NEXT:    addv s0, v0.4s4394; CHECK-SD-NEXT:    fmov w8, s04395; CHECK-SD-NEXT:    add w0, w8, w24396; CHECK-SD-NEXT:    ret4397;4398; CHECK-GI-LABEL: test_usdot_v32i8:4399; CHECK-GI:       // %bb.0: // %entry4400; CHECK-GI-NEXT:    ldp q0, q1, [x1]4401; CHECK-GI-NEXT:    ldp q2, q3, [x0]4402; CHECK-GI-NEXT:    sshll v4.8h, v0.8b, #04403; CHECK-GI-NEXT:    sshll2 v0.8h, v0.16b, #04404; CHECK-GI-NEXT:    sshll v5.8h, v1.8b, #04405; CHECK-GI-NEXT:    sshll2 v1.8h, v1.16b, #04406; CHECK-GI-NEXT:    ushll v6.8h, v2.8b, #04407; CHECK-GI-NEXT:    ushll2 v2.8h, v2.16b, #04408; CHECK-GI-NEXT:    ushll v7.8h, v3.8b, #04409; CHECK-GI-NEXT:    ushll2 v3.8h, v3.16b, #04410; CHECK-GI-NEXT:    smull v16.4s, v4.4h, v6.4h4411; CHECK-GI-NEXT:    smull v17.4s, v0.4h, v2.4h4412; CHECK-GI-NEXT:    smull v18.4s, v5.4h, v7.4h4413; CHECK-GI-NEXT:    smull v19.4s, v1.4h, v3.4h4414; CHECK-GI-NEXT:    smlal2 v16.4s, v4.8h, v6.8h4415; CHECK-GI-NEXT:    smlal2 v17.4s, v0.8h, v2.8h4416; CHECK-GI-NEXT:    smlal2 v18.4s, v5.8h, v7.8h4417; CHECK-GI-NEXT:    smlal2 v19.4s, v1.8h, v3.8h4418; CHECK-GI-NEXT:    add v0.4s, v16.4s, v17.4s4419; CHECK-GI-NEXT:    add v1.4s, v18.4s, v19.4s4420; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s4421; CHECK-GI-NEXT:    addv s0, v0.4s4422; CHECK-GI-NEXT:    fmov w8, s04423; CHECK-GI-NEXT:    add w0, w8, w24424; CHECK-GI-NEXT:    ret4425entry:4426  %0 = load <32 x i8>, ptr %a4427  %1 = zext <32 x i8> %0 to <32 x i32>4428  %2 = load <32 x i8>, ptr %b4429  %3 = sext <32 x i8> %2 to <32 x i32>4430  %4 = mul nsw <32 x i32> %3, %14431  %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4)4432  %op.extra = add nsw i32 %5, %sum4433  ret i32 %op.extra4434}4435 4436define i32 @test_usdot_v32i8_double(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {4437; CHECK-SD-LABEL: test_usdot_v32i8_double:4438; CHECK-SD:       // %bb.0: // %entry4439; CHECK-SD-NEXT:    movi v16.2d, #00000000000000004440; CHECK-SD-NEXT:    movi v17.2d, #00000000000000004441; CHECK-SD-NEXT:    movi v18.2d, #00000000000000004442; CHECK-SD-NEXT:    movi v19.2d, #00000000000000004443; CHECK-SD-NEXT:    usdot v16.4s, v1.16b, v3.16b4444; CHECK-SD-NEXT:    usdot v18.4s, v0.16b, v2.16b4445; CHECK-SD-NEXT:    usdot v17.4s, v4.16b, v6.16b4446; CHECK-SD-NEXT:    usdot v19.4s, v5.16b, v7.16b4447; CHECK-SD-NEXT:    add v0.4s, v18.4s, v16.4s4448; CHECK-SD-NEXT:    add v1.4s, v17.4s, v19.4s4449; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s4450; CHECK-SD-NEXT:    addv s0, v0.4s4451; CHECK-SD-NEXT:    fmov w0, s04452; CHECK-SD-NEXT:    ret4453;4454; CHECK-GI-LABEL: test_usdot_v32i8_double:4455; CHECK-GI:       // %bb.0: // %entry4456; CHECK-GI-NEXT:    ushll v16.8h, v0.8b, #04457; CHECK-GI-NEXT:    ushll2 v0.8h, v0.16b, #04458; CHECK-GI-NEXT:    ushll v17.8h, v1.8b, #04459; CHECK-GI-NEXT:    ushll2 v1.8h, v1.16b, #04460; CHECK-GI-NEXT:    sshll v18.8h, v2.8b, #04461; CHECK-GI-NEXT:    sshll2 v2.8h, v2.16b, #04462; CHECK-GI-NEXT:    sshll v19.8h, v3.8b, #04463; CHECK-GI-NEXT:    sshll2 v3.8h, v3.16b, #04464; CHECK-GI-NEXT:    ushll v20.8h, v4.8b, #04465; CHECK-GI-NEXT:    ushll2 v4.8h, v4.16b, #04466; CHECK-GI-NEXT:    ushll v21.8h, v5.8b, #04467; CHECK-GI-NEXT:    ushll2 v5.8h, v5.16b, #04468; CHECK-GI-NEXT:    sshll v22.8h, v6.8b, #04469; CHECK-GI-NEXT:    sshll2 v6.8h, v6.16b, #04470; CHECK-GI-NEXT:    sshll v23.8h, v7.8b, #04471; CHECK-GI-NEXT:    sshll2 v7.8h, v7.16b, #04472; CHECK-GI-NEXT:    smull v24.4s, v16.4h, v18.4h4473; CHECK-GI-NEXT:    smull v25.4s, v0.4h, v2.4h4474; CHECK-GI-NEXT:    smull v26.4s, v17.4h, v19.4h4475; CHECK-GI-NEXT:    smull v27.4s, v1.4h, v3.4h4476; CHECK-GI-NEXT:    smull v28.4s, v20.4h, v22.4h4477; CHECK-GI-NEXT:    smull v29.4s, v4.4h, v6.4h4478; CHECK-GI-NEXT:    smull v30.4s, v21.4h, v23.4h4479; CHECK-GI-NEXT:    smull v31.4s, v5.4h, v7.4h4480; CHECK-GI-NEXT:    smlal2 v24.4s, v16.8h, v18.8h4481; CHECK-GI-NEXT:    smlal2 v25.4s, v0.8h, v2.8h4482; CHECK-GI-NEXT:    smlal2 v26.4s, v17.8h, v19.8h4483; CHECK-GI-NEXT:    smlal2 v27.4s, v1.8h, v3.8h4484; CHECK-GI-NEXT:    smlal2 v28.4s, v20.8h, v22.8h4485; CHECK-GI-NEXT:    smlal2 v29.4s, v4.8h, v6.8h4486; CHECK-GI-NEXT:    smlal2 v30.4s, v21.8h, v23.8h4487; CHECK-GI-NEXT:    smlal2 v31.4s, v5.8h, v7.8h4488; CHECK-GI-NEXT:    add v0.4s, v24.4s, v25.4s4489; CHECK-GI-NEXT:    add v1.4s, v26.4s, v27.4s4490; CHECK-GI-NEXT:    add v2.4s, v28.4s, v29.4s4491; CHECK-GI-NEXT:    add v3.4s, v30.4s, v31.4s4492; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s4493; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s4494; CHECK-GI-NEXT:    addv s0, v0.4s4495; CHECK-GI-NEXT:    addv s1, v1.4s4496; CHECK-GI-NEXT:    fmov w8, s04497; CHECK-GI-NEXT:    fmov w9, s14498; CHECK-GI-NEXT:    add w0, w8, w94499; CHECK-GI-NEXT:    ret4500entry:4501  %az = zext <32 x i8> %a to <32 x i32>4502  %bz = sext <32 x i8> %b to <32 x i32>4503  %m1 = mul nuw nsw <32 x i32> %az, %bz4504  %r1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m1)4505  %cz = zext <32 x i8> %c to <32 x i32>4506  %dz = sext <32 x i8> %d to <32 x i32>4507  %m2 = mul nuw nsw <32 x i32> %cz, %dz4508  %r2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m2)4509  %x = add i32 %r1, %r24510  ret i32 %x4511}4512 4513 4514define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {4515; CHECK-SD-LABEL: test_udot_v33i8:4516; CHECK-SD:       // %bb.0: // %entry4517; CHECK-SD-NEXT:    ldr b0, [x0, #32]4518; CHECK-SD-NEXT:    ldr b1, [x1, #32]4519; CHECK-SD-NEXT:    movi v7.2d, #00000000000000004520; CHECK-SD-NEXT:    ldp q3, q4, [x1]4521; CHECK-SD-NEXT:    umull v0.8h, v1.8b, v0.8b4522; CHECK-SD-NEXT:    ldp q1, q2, [x0]4523; CHECK-SD-NEXT:    umull v5.8h, v4.8b, v2.8b4524; CHECK-SD-NEXT:    umull v6.8h, v3.8b, v1.8b4525; CHECK-SD-NEXT:    umull2 v2.8h, v4.16b, v2.16b4526; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #04527; CHECK-SD-NEXT:    umull2 v1.8h, v3.16b, v1.16b4528; CHECK-SD-NEXT:    mov v7.s[0], v0.s[0]4529; CHECK-SD-NEXT:    uaddl2 v3.4s, v6.8h, v5.8h4530; CHECK-SD-NEXT:    uaddl2 v0.4s, v1.8h, v2.8h4531; CHECK-SD-NEXT:    uaddl v1.4s, v1.4h, v2.4h4532; CHECK-SD-NEXT:    add v0.4s, v3.4s, v0.4s4533; CHECK-SD-NEXT:    uaddw v2.4s, v7.4s, v6.4h4534; CHECK-SD-NEXT:    uaddw v2.4s, v2.4s, v5.4h4535; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s4536; CHECK-SD-NEXT:    add v0.4s, v2.4s, v0.4s4537; CHECK-SD-NEXT:    addv s0, v0.4s4538; CHECK-SD-NEXT:    fmov w8, s04539; CHECK-SD-NEXT:    add w0, w8, w24540; CHECK-SD-NEXT:    ret4541;4542; CHECK-GI-LABEL: test_udot_v33i8:4543; CHECK-GI:       // %bb.0: // %entry4544; CHECK-GI-NEXT:    sub sp, sp, #1124545; CHECK-GI-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill4546; CHECK-GI-NEXT:    stp x28, x27, [sp, #32] // 16-byte Folded Spill4547; CHECK-GI-NEXT:    stp x26, x25, [sp, #48] // 16-byte Folded Spill4548; CHECK-GI-NEXT:    stp x24, x23, [sp, #64] // 16-byte Folded Spill4549; CHECK-GI-NEXT:    stp x22, x21, [sp, #80] // 16-byte Folded Spill4550; CHECK-GI-NEXT:    stp x20, x19, [sp, #96] // 16-byte Folded Spill4551; CHECK-GI-NEXT:    .cfi_def_cfa_offset 1124552; CHECK-GI-NEXT:    .cfi_offset w19, -84553; CHECK-GI-NEXT:    .cfi_offset w20, -164554; CHECK-GI-NEXT:    .cfi_offset w21, -244555; CHECK-GI-NEXT:    .cfi_offset w22, -324556; CHECK-GI-NEXT:    .cfi_offset w23, -404557; CHECK-GI-NEXT:    .cfi_offset w24, -484558; CHECK-GI-NEXT:    .cfi_offset w25, -564559; CHECK-GI-NEXT:    .cfi_offset w26, -644560; CHECK-GI-NEXT:    .cfi_offset w27, -724561; CHECK-GI-NEXT:    .cfi_offset w28, -804562; CHECK-GI-NEXT:    .cfi_offset w30, -884563; CHECK-GI-NEXT:    .cfi_offset w29, -964564; CHECK-GI-NEXT:    ldp q7, q16, [x1]4565; CHECK-GI-NEXT:    movi d5, #00000000000000004566; CHECK-GI-NEXT:    str w2, [sp, #12] // 4-byte Spill4567; CHECK-GI-NEXT:    movi d6, #00000000000000004568; CHECK-GI-NEXT:    movi d0, #00000000000000004569; CHECK-GI-NEXT:    movi d1, #00000000000000004570; CHECK-GI-NEXT:    movi d3, #00000000000000004571; CHECK-GI-NEXT:    movi d2, #00000000000000004572; CHECK-GI-NEXT:    mov b23, v7.b[7]4573; CHECK-GI-NEXT:    mov b17, v7.b[1]4574; CHECK-GI-NEXT:    fmov w11, s74575; CHECK-GI-NEXT:    mov b18, v7.b[2]4576; CHECK-GI-NEXT:    mov b19, v7.b[3]4577; CHECK-GI-NEXT:    mov b20, v7.b[4]4578; CHECK-GI-NEXT:    mov b21, v7.b[5]4579; CHECK-GI-NEXT:    mov b22, v7.b[6]4580; CHECK-GI-NEXT:    mov b24, v7.b[8]4581; CHECK-GI-NEXT:    uxtb w11, w114582; CHECK-GI-NEXT:    mov b25, v7.b[9]4583; CHECK-GI-NEXT:    mov b26, v7.b[10]4584; CHECK-GI-NEXT:    mov b27, v7.b[11]4585; CHECK-GI-NEXT:    mov b28, v7.b[12]4586; CHECK-GI-NEXT:    mov b29, v7.b[13]4587; CHECK-GI-NEXT:    mov b30, v7.b[14]4588; CHECK-GI-NEXT:    mov b7, v7.b[15]4589; CHECK-GI-NEXT:    fmov w7, s234590; CHECK-GI-NEXT:    mov b23, v16.b[7]4591; CHECK-GI-NEXT:    fmov w10, s174592; CHECK-GI-NEXT:    fmov w9, s184593; CHECK-GI-NEXT:    fmov w13, s194594; CHECK-GI-NEXT:    fmov w8, s244595; CHECK-GI-NEXT:    mov b17, v16.b[2]4596; CHECK-GI-NEXT:    fmov w12, s204597; CHECK-GI-NEXT:    fmov w16, s254598; CHECK-GI-NEXT:    fmov w23, s214599; CHECK-GI-NEXT:    uxtb w10, w104600; CHECK-GI-NEXT:    uxtb w9, w94601; CHECK-GI-NEXT:    mov b18, v16.b[1]4602; CHECK-GI-NEXT:    stp s23, s7, [sp, #4] // 8-byte Folded Spill4603; CHECK-GI-NEXT:    uxtb w8, w84604; CHECK-GI-NEXT:    fmov s7, w114605; CHECK-GI-NEXT:    fmov w5, s174606; CHECK-GI-NEXT:    fmov w27, s264607; CHECK-GI-NEXT:    mov b21, v16.b[5]4608; CHECK-GI-NEXT:    fmov s17, w84609; CHECK-GI-NEXT:    uxtb w8, w124610; CHECK-GI-NEXT:    fmov w20, s224611; CHECK-GI-NEXT:    mov v7.h[1], w104612; CHECK-GI-NEXT:    uxtb w10, w164613; CHECK-GI-NEXT:    mov b19, v16.b[3]4614; CHECK-GI-NEXT:    mov b22, v16.b[4]4615; CHECK-GI-NEXT:    mov b20, v16.b[6]4616; CHECK-GI-NEXT:    fmov w21, s274617; CHECK-GI-NEXT:    mov v17.h[1], w104618; CHECK-GI-NEXT:    fmov w24, s284619; CHECK-GI-NEXT:    mov b24, v16.b[8]4620; CHECK-GI-NEXT:    fmov w22, s294621; CHECK-GI-NEXT:    mov b26, v16.b[9]4622; CHECK-GI-NEXT:    fmov w4, s304623; CHECK-GI-NEXT:    uxtb w10, w214624; CHECK-GI-NEXT:    mov v7.h[2], w94625; CHECK-GI-NEXT:    uxtb w9, w134626; CHECK-GI-NEXT:    str s20, [sp] // 4-byte Spill4627; CHECK-GI-NEXT:    mov b25, v16.b[10]4628; CHECK-GI-NEXT:    fmov w25, s184629; CHECK-GI-NEXT:    uxtb w22, w224630; CHECK-GI-NEXT:    mov b27, v16.b[11]4631; CHECK-GI-NEXT:    mov b28, v16.b[12]4632; CHECK-GI-NEXT:    mov b29, v16.b[13]4633; CHECK-GI-NEXT:    mov b30, v16.b[14]4634; CHECK-GI-NEXT:    fmov w26, s164635; CHECK-GI-NEXT:    mov v7.h[3], w94636; CHECK-GI-NEXT:    uxtb w9, w274637; CHECK-GI-NEXT:    mov b31, v16.b[15]4638; CHECK-GI-NEXT:    ldp q18, q16, [x0]4639; CHECK-GI-NEXT:    fmov w2, s214640; CHECK-GI-NEXT:    uxtb w26, w264641; CHECK-GI-NEXT:    mov v17.h[2], w94642; CHECK-GI-NEXT:    fmov w14, s224643; CHECK-GI-NEXT:    fmov w3, s254644; CHECK-GI-NEXT:    fmov w15, s194645; CHECK-GI-NEXT:    fmov w19, s244646; CHECK-GI-NEXT:    mov v7.h[4], w84647; CHECK-GI-NEXT:    uxtb w8, w234648; CHECK-GI-NEXT:    mov b21, v18.b[2]4649; CHECK-GI-NEXT:    mov b22, v18.b[1]4650; CHECK-GI-NEXT:    mov b25, v18.b[5]4651; CHECK-GI-NEXT:    mov b23, v18.b[6]4652; CHECK-GI-NEXT:    uxtb w19, w194653; CHECK-GI-NEXT:    uxtb w3, w34654; CHECK-GI-NEXT:    mov v17.h[3], w104655; CHECK-GI-NEXT:    uxtb w10, w244656; CHECK-GI-NEXT:    uxtb w24, w74657; CHECK-GI-NEXT:    mov b19, v18.b[3]4658; CHECK-GI-NEXT:    mov v7.h[5], w84659; CHECK-GI-NEXT:    uxtb w8, w204660; CHECK-GI-NEXT:    fmov w29, s214661; CHECK-GI-NEXT:    mov b21, v18.b[10]4662; CHECK-GI-NEXT:    fmov w9, s224663; CHECK-GI-NEXT:    fmov w6, s264664; CHECK-GI-NEXT:    mov v17.h[4], w104665; CHECK-GI-NEXT:    uxtb w10, w254666; CHECK-GI-NEXT:    fmov w17, s274667; CHECK-GI-NEXT:    mov b26, v18.b[4]4668; CHECK-GI-NEXT:    fmov w18, s284669; CHECK-GI-NEXT:    fmov w16, s294670; CHECK-GI-NEXT:    mov v7.h[6], w84671; CHECK-GI-NEXT:    fmov w8, s184672; CHECK-GI-NEXT:    mov b24, v18.b[7]4673; CHECK-GI-NEXT:    fmov w30, s214674; CHECK-GI-NEXT:    mov b20, v18.b[8]4675; CHECK-GI-NEXT:    mov b27, v18.b[9]4676; CHECK-GI-NEXT:    uxtb w16, w164677; CHECK-GI-NEXT:    mov b28, v18.b[11]4678; CHECK-GI-NEXT:    mov b29, v18.b[12]4679; CHECK-GI-NEXT:    fmov w23, s254680; CHECK-GI-NEXT:    mov b25, v18.b[13]4681; CHECK-GI-NEXT:    fmov w21, s234682; CHECK-GI-NEXT:    mov v7.h[7], w244683; CHECK-GI-NEXT:    uxtb w24, w84684; CHECK-GI-NEXT:    uxtb w8, w94685; CHECK-GI-NEXT:    uxtb w9, w294686; CHECK-GI-NEXT:    mov b23, v18.b[14]4687; CHECK-GI-NEXT:    mov b22, v18.b[15]4688; CHECK-GI-NEXT:    fmov s21, w244689; CHECK-GI-NEXT:    fmov s18, w264690; CHECK-GI-NEXT:    fmov w28, s194691; CHECK-GI-NEXT:    mov b19, v16.b[1]4692; CHECK-GI-NEXT:    mov v17.h[5], w224693; CHECK-GI-NEXT:    fmov w7, s204694; CHECK-GI-NEXT:    fmov w11, s274695; CHECK-GI-NEXT:    fmov w27, s264696; CHECK-GI-NEXT:    mov b20, v16.b[2]4697; CHECK-GI-NEXT:    mov v21.h[1], w84698; CHECK-GI-NEXT:    uxtb w8, w44699; CHECK-GI-NEXT:    mov v18.h[1], w104700; CHECK-GI-NEXT:    uxtb w10, w54701; CHECK-GI-NEXT:    uxtb w7, w74702; CHECK-GI-NEXT:    fmov w24, s234703; CHECK-GI-NEXT:    mov b23, v16.b[6]4704; CHECK-GI-NEXT:    fmov w4, s224705; CHECK-GI-NEXT:    mov b22, v16.b[8]4706; CHECK-GI-NEXT:    mov v17.h[6], w84707; CHECK-GI-NEXT:    fmov w8, s194708; CHECK-GI-NEXT:    fmov s19, w194709; CHECK-GI-NEXT:    mov v21.h[2], w94710; CHECK-GI-NEXT:    uxtb w9, w284711; CHECK-GI-NEXT:    mov v18.h[2], w104712; CHECK-GI-NEXT:    uxtb w10, w64713; CHECK-GI-NEXT:    mov b27, v16.b[9]4714; CHECK-GI-NEXT:    fmov w20, s244715; CHECK-GI-NEXT:    uxtb w8, w84716; CHECK-GI-NEXT:    mov b24, v16.b[3]4717; CHECK-GI-NEXT:    fmov w5, s204718; CHECK-GI-NEXT:    mov v19.h[1], w104719; CHECK-GI-NEXT:    fmov w10, s234720; CHECK-GI-NEXT:    fmov s23, w74721; CHECK-GI-NEXT:    mov v21.h[3], w94722; CHECK-GI-NEXT:    uxtb w9, w114723; CHECK-GI-NEXT:    uxtb w11, w274724; CHECK-GI-NEXT:    uxtb w27, w304725; CHECK-GI-NEXT:    uxtb w5, w54726; CHECK-GI-NEXT:    fmov w7, s224727; CHECK-GI-NEXT:    uxtb w10, w104728; CHECK-GI-NEXT:    mov v23.h[1], w94729; CHECK-GI-NEXT:    fmov w9, s164730; CHECK-GI-NEXT:    mov b20, v16.b[10]4731; CHECK-GI-NEXT:    fmov w22, s284732; CHECK-GI-NEXT:    fmov w25, s254733; CHECK-GI-NEXT:    uxtb w7, w74734; CHECK-GI-NEXT:    mov v21.h[4], w114735; CHECK-GI-NEXT:    fmov w11, s274736; CHECK-GI-NEXT:    uxtb w9, w94737; CHECK-GI-NEXT:    mov b25, v16.b[5]4738; CHECK-GI-NEXT:    fmov w29, s244739; CHECK-GI-NEXT:    fmov s22, w74740; CHECK-GI-NEXT:    mov v23.h[2], w274741; CHECK-GI-NEXT:    mov b24, v16.b[11]4742; CHECK-GI-NEXT:    uxtb w11, w114743; CHECK-GI-NEXT:    fmov w27, s204744; CHECK-GI-NEXT:    fmov s20, w94745; CHECK-GI-NEXT:    fmov w26, s294746; CHECK-GI-NEXT:    mov b26, v16.b[4]4747; CHECK-GI-NEXT:    mov v19.h[2], w34748; CHECK-GI-NEXT:    uxtb w3, w294749; CHECK-GI-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload4750; CHECK-GI-NEXT:    mov v22.h[1], w114751; CHECK-GI-NEXT:    uxtb w11, w154752; CHECK-GI-NEXT:    uxtb w15, w224753; CHECK-GI-NEXT:    uxtb w22, w234754; CHECK-GI-NEXT:    mov v20.h[1], w84755; CHECK-GI-NEXT:    fmov w6, s254756; CHECK-GI-NEXT:    mov v18.h[3], w114757; CHECK-GI-NEXT:    uxtb w11, w274758; CHECK-GI-NEXT:    mov v23.h[3], w154759; CHECK-GI-NEXT:    uxtb w15, w174760; CHECK-GI-NEXT:    uxtb w17, w214761; CHECK-GI-NEXT:    mov b25, v16.b[12]4762; CHECK-GI-NEXT:    fmov w28, s244763; CHECK-GI-NEXT:    mov v21.h[5], w224764; CHECK-GI-NEXT:    mov v22.h[2], w114765; CHECK-GI-NEXT:    uxtb w11, w144766; CHECK-GI-NEXT:    uxtb w14, w264767; CHECK-GI-NEXT:    mov v20.h[2], w54768; CHECK-GI-NEXT:    ldp x22, x21, [sp, #80] // 16-byte Folded Reload4769; CHECK-GI-NEXT:    fmov w19, s264770; CHECK-GI-NEXT:    mov v18.h[4], w114771; CHECK-GI-NEXT:    uxtb w11, w284772; CHECK-GI-NEXT:    mov v23.h[4], w144773; CHECK-GI-NEXT:    uxtb w14, w254774; CHECK-GI-NEXT:    ldp x26, x25, [sp, #48] // 16-byte Folded Reload4775; CHECK-GI-NEXT:    ldp x28, x27, [sp, #32] // 16-byte Folded Reload4776; CHECK-GI-NEXT:    mov b26, v16.b[13]4777; CHECK-GI-NEXT:    fmov w7, s254778; CHECK-GI-NEXT:    mov v19.h[3], w154779; CHECK-GI-NEXT:    uxtb w15, w184780; CHECK-GI-NEXT:    uxtb w18, w194781; CHECK-GI-NEXT:    mov v21.h[6], w174782; CHECK-GI-NEXT:    uxtb w17, w204783; CHECK-GI-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload4784; CHECK-GI-NEXT:    mov v22.h[3], w114785; CHECK-GI-NEXT:    uxtb w11, w24786; CHECK-GI-NEXT:    mov v20.h[3], w34787; CHECK-GI-NEXT:    mov v23.h[5], w144788; CHECK-GI-NEXT:    uxtb w14, w244789; CHECK-GI-NEXT:    ldp x24, x23, [sp, #64] // 16-byte Folded Reload4790; CHECK-GI-NEXT:    mov v18.h[5], w114791; CHECK-GI-NEXT:    uxtb w11, w74792; CHECK-GI-NEXT:    fmov w8, s264793; CHECK-GI-NEXT:    mov v19.h[4], w154794; CHECK-GI-NEXT:    ldr w15, [sp] // 4-byte Reload4795; CHECK-GI-NEXT:    mov v21.h[7], w174796; CHECK-GI-NEXT:    uxtb w17, w64797; CHECK-GI-NEXT:    mov v22.h[4], w114798; CHECK-GI-NEXT:    ldr w11, [sp, #8] // 4-byte Reload4799; CHECK-GI-NEXT:    uxtb w8, w84800; CHECK-GI-NEXT:    uxtb w15, w154801; CHECK-GI-NEXT:    fmov w13, s304802; CHECK-GI-NEXT:    uxtb w11, w114803; CHECK-GI-NEXT:    mov v20.h[4], w184804; CHECK-GI-NEXT:    mov v23.h[6], w144805; CHECK-GI-NEXT:    mov v19.h[5], w164806; CHECK-GI-NEXT:    mov b27, v16.b[14]4807; CHECK-GI-NEXT:    mul v24.8h, v7.8h, v21.8h4808; CHECK-GI-NEXT:    mov v22.h[5], w84809; CHECK-GI-NEXT:    uxtb w8, w44810; CHECK-GI-NEXT:    mov b7, v16.b[7]4811; CHECK-GI-NEXT:    mov b16, v16.b[15]4812; CHECK-GI-NEXT:    fmov w12, s314813; CHECK-GI-NEXT:    mov v17.h[7], w114814; CHECK-GI-NEXT:    uxtb w11, w134815; CHECK-GI-NEXT:    ldr w13, [sp, #4] // 4-byte Reload4816; CHECK-GI-NEXT:    mov v20.h[5], w174817; CHECK-GI-NEXT:    mov v23.h[7], w84818; CHECK-GI-NEXT:    fmov w9, s274819; CHECK-GI-NEXT:    mov v18.h[6], w154820; CHECK-GI-NEXT:    uxtb w8, w124821; CHECK-GI-NEXT:    uxtb w13, w134822; CHECK-GI-NEXT:    mov v19.h[6], w114823; CHECK-GI-NEXT:    fmov w12, s164824; CHECK-GI-NEXT:    fmov w11, s74825; CHECK-GI-NEXT:    movi d4, #00000000000000004826; CHECK-GI-NEXT:    uxtb w9, w94827; CHECK-GI-NEXT:    mov v20.h[6], w104828; CHECK-GI-NEXT:    umov w10, v24.h[0]4829; CHECK-GI-NEXT:    mul v21.8h, v17.8h, v23.8h4830; CHECK-GI-NEXT:    mov v18.h[7], w134831; CHECK-GI-NEXT:    mov v5.s[1], wzr4832; CHECK-GI-NEXT:    uxtb w11, w114833; CHECK-GI-NEXT:    mov v19.h[7], w84834; CHECK-GI-NEXT:    uxtb w8, w124835; CHECK-GI-NEXT:    umov w12, v24.h[4]4836; CHECK-GI-NEXT:    mov v22.h[6], w94837; CHECK-GI-NEXT:    umov w9, v24.h[1]4838; CHECK-GI-NEXT:    mov v20.h[7], w114839; CHECK-GI-NEXT:    umov w11, v24.h[5]4840; CHECK-GI-NEXT:    fmov s7, w104841; CHECK-GI-NEXT:    ldrb w10, [x1, #32]4842; CHECK-GI-NEXT:    umov w13, v21.h[0]4843; CHECK-GI-NEXT:    umov w14, v21.h[1]4844; CHECK-GI-NEXT:    umov w15, v21.h[4]4845; CHECK-GI-NEXT:    mov v6.s[1], wzr4846; CHECK-GI-NEXT:    mov v0.s[1], wzr4847; CHECK-GI-NEXT:    fmov s16, w124848; CHECK-GI-NEXT:    mov v22.h[7], w84849; CHECK-GI-NEXT:    umov w12, v24.h[6]4850; CHECK-GI-NEXT:    umov w8, v24.h[2]4851; CHECK-GI-NEXT:    mov v7.s[1], w94852; CHECK-GI-NEXT:    ldrb w9, [x0, #32]4853; CHECK-GI-NEXT:    fmov s17, w134854; CHECK-GI-NEXT:    mul v23.8h, v18.8h, v20.8h4855; CHECK-GI-NEXT:    umov w13, v24.h[7]4856; CHECK-GI-NEXT:    mov v16.s[1], w114857; CHECK-GI-NEXT:    umov w11, v21.h[5]4858; CHECK-GI-NEXT:    fmov s18, w154859; CHECK-GI-NEXT:    mul v19.8h, v19.8h, v22.8h4860; CHECK-GI-NEXT:    umov w15, v21.h[6]4861; CHECK-GI-NEXT:    mov v1.s[1], wzr4862; CHECK-GI-NEXT:    mov v17.s[1], w144863; CHECK-GI-NEXT:    umov w14, v21.h[2]4864; CHECK-GI-NEXT:    mov v7.s[2], w84865; CHECK-GI-NEXT:    mul w8, w10, w94866; CHECK-GI-NEXT:    umov w9, v23.h[0]4867; CHECK-GI-NEXT:    umov w10, v23.h[1]4868; CHECK-GI-NEXT:    mov v16.s[2], w124869; CHECK-GI-NEXT:    umov w12, v21.h[3]4870; CHECK-GI-NEXT:    mov v18.s[1], w114871; CHECK-GI-NEXT:    umov w11, v23.h[4]4872; CHECK-GI-NEXT:    mov v3.s[1], wzr4873; CHECK-GI-NEXT:    mov v2.s[1], wzr4874; CHECK-GI-NEXT:    mov v17.s[2], w144875; CHECK-GI-NEXT:    umov w14, v23.h[5]4876; CHECK-GI-NEXT:    mov v4.s[1], wzr4877; CHECK-GI-NEXT:    fmov s20, w94878; CHECK-GI-NEXT:    umov w9, v19.h[1]4879; CHECK-GI-NEXT:    mov v5.s[2], wzr4880; CHECK-GI-NEXT:    mov v16.s[3], w134881; CHECK-GI-NEXT:    umov w13, v19.h[0]4882; CHECK-GI-NEXT:    mov v18.s[2], w154883; CHECK-GI-NEXT:    umov w15, v21.h[7]4884; CHECK-GI-NEXT:    fmov s21, w114885; CHECK-GI-NEXT:    umov w11, v23.h[2]4886; CHECK-GI-NEXT:    mov v17.s[3], w124887; CHECK-GI-NEXT:    umov w12, v19.h[4]4888; CHECK-GI-NEXT:    mov v20.s[1], w104889; CHECK-GI-NEXT:    umov w10, v23.h[3]4890; CHECK-GI-NEXT:    mov v6.s[2], wzr4891; CHECK-GI-NEXT:    umov w16, v24.h[3]4892; CHECK-GI-NEXT:    fmov s22, w134893; CHECK-GI-NEXT:    umov w13, v19.h[5]4894; CHECK-GI-NEXT:    mov v21.s[1], w144895; CHECK-GI-NEXT:    umov w14, v23.h[6]4896; CHECK-GI-NEXT:    mov v18.s[3], w154897; CHECK-GI-NEXT:    umov w15, v19.h[2]4898; CHECK-GI-NEXT:    mov v20.s[2], w114899; CHECK-GI-NEXT:    umov w11, v19.h[6]4900; CHECK-GI-NEXT:    mov v0.s[2], wzr4901; CHECK-GI-NEXT:    mov v22.s[1], w94902; CHECK-GI-NEXT:    umov w9, v23.h[7]4903; CHECK-GI-NEXT:    fmov s23, w124904; CHECK-GI-NEXT:    umov w12, v19.h[3]4905; CHECK-GI-NEXT:    mov v1.s[2], wzr4906; CHECK-GI-NEXT:    mov v3.s[2], wzr4907; CHECK-GI-NEXT:    mov v21.s[2], w144908; CHECK-GI-NEXT:    mov v2.s[2], wzr4909; CHECK-GI-NEXT:    mov v4.s[2], wzr4910; CHECK-GI-NEXT:    mov v23.s[1], w134911; CHECK-GI-NEXT:    mov v5.s[3], wzr4912; CHECK-GI-NEXT:    mov v6.s[3], wzr4913; CHECK-GI-NEXT:    mov v22.s[2], w154914; CHECK-GI-NEXT:    mov v7.s[3], w164915; CHECK-GI-NEXT:    mov v20.s[3], w104916; CHECK-GI-NEXT:    mov v0.s[3], wzr4917; CHECK-GI-NEXT:    mov v1.s[3], wzr4918; CHECK-GI-NEXT:    mov v3.s[3], wzr4919; CHECK-GI-NEXT:    mov v21.s[3], w94920; CHECK-GI-NEXT:    ldr w9, [sp, #12] // 4-byte Reload4921; CHECK-GI-NEXT:    mov v2.s[3], wzr4922; CHECK-GI-NEXT:    mov v23.s[2], w114923; CHECK-GI-NEXT:    umov w11, v19.h[7]4924; CHECK-GI-NEXT:    fmov s19, w84925; CHECK-GI-NEXT:    mov v22.s[3], w124926; CHECK-GI-NEXT:    mov v4.s[3], wzr4927; CHECK-GI-NEXT:    add v5.4s, v5.4s, v6.4s4928; CHECK-GI-NEXT:    add v6.4s, v7.4s, v16.4s4929; CHECK-GI-NEXT:    add v7.4s, v17.4s, v18.4s4930; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s4931; CHECK-GI-NEXT:    mov v19.s[1], wzr4932; CHECK-GI-NEXT:    add v16.4s, v20.4s, v21.4s4933; CHECK-GI-NEXT:    mov v23.s[3], w114934; CHECK-GI-NEXT:    add v0.4s, v0.4s, v5.4s4935; CHECK-GI-NEXT:    add v2.4s, v2.4s, v4.4s4936; CHECK-GI-NEXT:    add v3.4s, v6.4s, v7.4s4937; CHECK-GI-NEXT:    mov v19.s[2], wzr4938; CHECK-GI-NEXT:    add v17.4s, v22.4s, v23.4s4939; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s4940; CHECK-GI-NEXT:    mov v19.s[3], wzr4941; CHECK-GI-NEXT:    add v4.4s, v16.4s, v17.4s4942; CHECK-GI-NEXT:    add v2.4s, v3.4s, v4.4s4943; CHECK-GI-NEXT:    add v0.4s, v19.4s, v0.4s4944; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s4945; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s4946; CHECK-GI-NEXT:    addv s0, v0.4s4947; CHECK-GI-NEXT:    fmov w8, s04948; CHECK-GI-NEXT:    add w0, w8, w94949; CHECK-GI-NEXT:    add sp, sp, #1124950; CHECK-GI-NEXT:    ret4951entry:4952  %0 = load <33 x i8>, ptr %a4953  %1 = zext <33 x i8> %0 to <33 x i32>4954  %2 = load <33 x i8>, ptr %b4955  %3 = zext <33 x i8> %2 to <33 x i32>4956  %4 = mul nuw nsw <33 x i32> %3, %14957  %5 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %4)4958  %op.extra = add i32 %5, %sum4959  ret i32 %op.extra4960}4961 4962define i32 @test_udot_v33i8_nomla(ptr nocapture readonly %a1) {4963; CHECK-SD-LABEL: test_udot_v33i8_nomla:4964; CHECK-SD:       // %bb.0: // %entry4965; CHECK-SD-NEXT:    ldr b1, [x0, #32]4966; CHECK-SD-NEXT:    ldp q3, q2, [x0]4967; CHECK-SD-NEXT:    movi v0.2d, #00000000000000004968; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #04969; CHECK-SD-NEXT:    ushll v4.8h, v2.8b, #04970; CHECK-SD-NEXT:    ushll v5.8h, v3.8b, #04971; CHECK-SD-NEXT:    ushll2 v2.8h, v2.16b, #04972; CHECK-SD-NEXT:    ushll2 v3.8h, v3.16b, #04973; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #04974; CHECK-SD-NEXT:    uaddl2 v6.4s, v5.8h, v4.8h4975; CHECK-SD-NEXT:    mov v0.s[0], v1.s[0]4976; CHECK-SD-NEXT:    uaddl2 v1.4s, v3.8h, v2.8h4977; CHECK-SD-NEXT:    uaddl v2.4s, v3.4h, v2.4h4978; CHECK-SD-NEXT:    add v1.4s, v6.4s, v1.4s4979; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v5.4h4980; CHECK-SD-NEXT:    add v1.4s, v2.4s, v1.4s4981; CHECK-SD-NEXT:    uaddw v0.4s, v0.4s, v4.4h4982; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s4983; CHECK-SD-NEXT:    addv s0, v0.4s4984; CHECK-SD-NEXT:    fmov w0, s04985; CHECK-SD-NEXT:    ret4986;4987; CHECK-GI-LABEL: test_udot_v33i8_nomla:4988; CHECK-GI:       // %bb.0: // %entry4989; CHECK-GI-NEXT:    stp x20, x19, [sp, #-16]! // 16-byte Folded Spill4990; CHECK-GI-NEXT:    .cfi_def_cfa_offset 164991; CHECK-GI-NEXT:    .cfi_offset w19, -84992; CHECK-GI-NEXT:    .cfi_offset w20, -164993; CHECK-GI-NEXT:    ldp q7, q19, [x0]4994; CHECK-GI-NEXT:    movi d1, #00000000000000004995; CHECK-GI-NEXT:    ldrb w10, [x0, #32]4996; CHECK-GI-NEXT:    movi d0, #00000000000000004997; CHECK-GI-NEXT:    movi d3, #00000000000000004998; CHECK-GI-NEXT:    movi d2, #00000000000000004999; CHECK-GI-NEXT:    movi d5, #00000000000000005000; CHECK-GI-NEXT:    movi d4, #00000000000000005001; CHECK-GI-NEXT:    umov w15, v7.b[8]5002; CHECK-GI-NEXT:    umov w2, v7.b[12]5003; CHECK-GI-NEXT:    umov w16, v7.b[9]5004; CHECK-GI-NEXT:    umov w4, v7.b[13]5005; CHECK-GI-NEXT:    umov w8, v7.b[0]5006; CHECK-GI-NEXT:    umov w12, v7.b[4]5007; CHECK-GI-NEXT:    umov w9, v7.b[1]5008; CHECK-GI-NEXT:    umov w14, v7.b[5]5009; CHECK-GI-NEXT:    umov w3, v7.b[10]5010; CHECK-GI-NEXT:    umov w5, v7.b[14]5011; CHECK-GI-NEXT:    umov w11, v7.b[2]5012; CHECK-GI-NEXT:    umov w17, v7.b[3]5013; CHECK-GI-NEXT:    fmov s17, w155014; CHECK-GI-NEXT:    fmov s18, w25015; CHECK-GI-NEXT:    umov w13, v7.b[6]5016; CHECK-GI-NEXT:    umov w18, v7.b[7]5017; CHECK-GI-NEXT:    umov w0, v7.b[11]5018; CHECK-GI-NEXT:    umov w1, v7.b[15]5019; CHECK-GI-NEXT:    fmov s7, w85020; CHECK-GI-NEXT:    fmov s16, w125021; CHECK-GI-NEXT:    umov w2, v19.b[0]5022; CHECK-GI-NEXT:    mov v17.s[1], w165023; CHECK-GI-NEXT:    mov v18.s[1], w45024; CHECK-GI-NEXT:    umov w4, v19.b[4]5025; CHECK-GI-NEXT:    movi d6, #00000000000000005026; CHECK-GI-NEXT:    umov w6, v19.b[1]5027; CHECK-GI-NEXT:    umov w7, v19.b[5]5028; CHECK-GI-NEXT:    mov v7.s[1], w95029; CHECK-GI-NEXT:    mov v16.s[1], w145030; CHECK-GI-NEXT:    umov w19, v19.b[9]5031; CHECK-GI-NEXT:    umov w20, v19.b[13]5032; CHECK-GI-NEXT:    umov w12, v19.b[2]5033; CHECK-GI-NEXT:    umov w8, v19.b[3]5034; CHECK-GI-NEXT:    mov v17.s[2], w35035; CHECK-GI-NEXT:    umov w3, v19.b[8]5036; CHECK-GI-NEXT:    mov v18.s[2], w55037; CHECK-GI-NEXT:    umov w5, v19.b[12]5038; CHECK-GI-NEXT:    umov w14, v19.b[6]5039; CHECK-GI-NEXT:    umov w9, v19.b[7]5040; CHECK-GI-NEXT:    mov v7.s[2], w115041; CHECK-GI-NEXT:    mov v16.s[2], w135042; CHECK-GI-NEXT:    umov w15, v19.b[10]5043; CHECK-GI-NEXT:    umov w11, v19.b[11]5044; CHECK-GI-NEXT:    umov w16, v19.b[14]5045; CHECK-GI-NEXT:    umov w13, v19.b[15]5046; CHECK-GI-NEXT:    fmov s19, w25047; CHECK-GI-NEXT:    fmov s20, w45048; CHECK-GI-NEXT:    fmov s21, w35049; CHECK-GI-NEXT:    fmov s22, w55050; CHECK-GI-NEXT:    fmov s23, w105051; CHECK-GI-NEXT:    mov v1.s[1], wzr5052; CHECK-GI-NEXT:    mov v0.s[1], wzr5053; CHECK-GI-NEXT:    mov v3.s[1], wzr5054; CHECK-GI-NEXT:    mov v2.s[1], wzr5055; CHECK-GI-NEXT:    mov v19.s[1], w65056; CHECK-GI-NEXT:    mov v20.s[1], w75057; CHECK-GI-NEXT:    mov v21.s[1], w195058; CHECK-GI-NEXT:    mov v22.s[1], w205059; CHECK-GI-NEXT:    mov v23.s[1], wzr5060; CHECK-GI-NEXT:    mov v5.s[1], wzr5061; CHECK-GI-NEXT:    mov v4.s[1], wzr5062; CHECK-GI-NEXT:    mov v6.s[1], wzr5063; CHECK-GI-NEXT:    mov v1.s[2], wzr5064; CHECK-GI-NEXT:    mov v0.s[2], wzr5065; CHECK-GI-NEXT:    mov v3.s[2], wzr5066; CHECK-GI-NEXT:    mov v2.s[2], wzr5067; CHECK-GI-NEXT:    mov v19.s[2], w125068; CHECK-GI-NEXT:    mov v20.s[2], w145069; CHECK-GI-NEXT:    mov v21.s[2], w155070; CHECK-GI-NEXT:    mov v22.s[2], w165071; CHECK-GI-NEXT:    mov v23.s[2], wzr5072; CHECK-GI-NEXT:    mov v5.s[2], wzr5073; CHECK-GI-NEXT:    mov v4.s[2], wzr5074; CHECK-GI-NEXT:    mov v6.s[2], wzr5075; CHECK-GI-NEXT:    mov v7.s[3], w175076; CHECK-GI-NEXT:    mov v16.s[3], w185077; CHECK-GI-NEXT:    mov v17.s[3], w05078; CHECK-GI-NEXT:    mov v18.s[3], w15079; CHECK-GI-NEXT:    mov v19.s[3], w85080; CHECK-GI-NEXT:    mov v20.s[3], w95081; CHECK-GI-NEXT:    mov v21.s[3], w115082; CHECK-GI-NEXT:    mov v22.s[3], w135083; CHECK-GI-NEXT:    mov v23.s[3], wzr5084; CHECK-GI-NEXT:    mov v1.s[3], wzr5085; CHECK-GI-NEXT:    mov v0.s[3], wzr5086; CHECK-GI-NEXT:    mov v3.s[3], wzr5087; CHECK-GI-NEXT:    mov v2.s[3], wzr5088; CHECK-GI-NEXT:    mov v5.s[3], wzr5089; CHECK-GI-NEXT:    mov v4.s[3], wzr5090; CHECK-GI-NEXT:    mov v6.s[3], wzr5091; CHECK-GI-NEXT:    add v7.4s, v7.4s, v16.4s5092; CHECK-GI-NEXT:    add v16.4s, v17.4s, v18.4s5093; CHECK-GI-NEXT:    add v17.4s, v19.4s, v20.4s5094; CHECK-GI-NEXT:    add v18.4s, v21.4s, v22.4s5095; CHECK-GI-NEXT:    add v1.4s, v23.4s, v1.4s5096; CHECK-GI-NEXT:    add v0.4s, v0.4s, v3.4s5097; CHECK-GI-NEXT:    add v2.4s, v2.4s, v5.4s5098; CHECK-GI-NEXT:    add v3.4s, v4.4s, v6.4s5099; CHECK-GI-NEXT:    add v4.4s, v7.4s, v16.4s5100; CHECK-GI-NEXT:    add v5.4s, v17.4s, v18.4s5101; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s5102; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s5103; CHECK-GI-NEXT:    add v2.4s, v4.4s, v5.4s5104; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s5105; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s5106; CHECK-GI-NEXT:    addv s0, v0.4s5107; CHECK-GI-NEXT:    fmov w0, s05108; CHECK-GI-NEXT:    ldp x20, x19, [sp], #16 // 16-byte Folded Reload5109; CHECK-GI-NEXT:    ret5110entry:5111  %0 = load <33 x i8>, ptr %a15112  %1 = zext <33 x i8> %0 to <33 x i32>5113  %2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %1)5114  ret i32 %25115}5116define i32 @test_sdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {5117; CHECK-SD-LABEL: test_sdot_v33i8:5118; CHECK-SD:       // %bb.0: // %entry5119; CHECK-SD-NEXT:    ldr b0, [x0, #32]5120; CHECK-SD-NEXT:    ldr b1, [x1, #32]5121; CHECK-SD-NEXT:    movi v7.2d, #00000000000000005122; CHECK-SD-NEXT:    ldp q3, q4, [x1]5123; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b5124; CHECK-SD-NEXT:    ldp q1, q2, [x0]5125; CHECK-SD-NEXT:    smull v5.8h, v4.8b, v2.8b5126; CHECK-SD-NEXT:    smull v6.8h, v3.8b, v1.8b5127; CHECK-SD-NEXT:    smull2 v2.8h, v4.16b, v2.16b5128; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #05129; CHECK-SD-NEXT:    smull2 v1.8h, v3.16b, v1.16b5130; CHECK-SD-NEXT:    mov v7.s[0], v0.s[0]5131; CHECK-SD-NEXT:    saddl2 v3.4s, v6.8h, v5.8h5132; CHECK-SD-NEXT:    saddl2 v0.4s, v1.8h, v2.8h5133; CHECK-SD-NEXT:    saddl v1.4s, v1.4h, v2.4h5134; CHECK-SD-NEXT:    add v0.4s, v3.4s, v0.4s5135; CHECK-SD-NEXT:    saddw v2.4s, v7.4s, v6.4h5136; CHECK-SD-NEXT:    saddw v2.4s, v2.4s, v5.4h5137; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s5138; CHECK-SD-NEXT:    add v0.4s, v2.4s, v0.4s5139; CHECK-SD-NEXT:    addv s0, v0.4s5140; CHECK-SD-NEXT:    fmov w8, s05141; CHECK-SD-NEXT:    add w0, w8, w25142; CHECK-SD-NEXT:    ret5143;5144; CHECK-GI-LABEL: test_sdot_v33i8:5145; CHECK-GI:       // %bb.0: // %entry5146; CHECK-GI-NEXT:    sub sp, sp, #1125147; CHECK-GI-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill5148; CHECK-GI-NEXT:    stp x28, x27, [sp, #32] // 16-byte Folded Spill5149; CHECK-GI-NEXT:    stp x26, x25, [sp, #48] // 16-byte Folded Spill5150; CHECK-GI-NEXT:    stp x24, x23, [sp, #64] // 16-byte Folded Spill5151; CHECK-GI-NEXT:    stp x22, x21, [sp, #80] // 16-byte Folded Spill5152; CHECK-GI-NEXT:    stp x20, x19, [sp, #96] // 16-byte Folded Spill5153; CHECK-GI-NEXT:    .cfi_def_cfa_offset 1125154; CHECK-GI-NEXT:    .cfi_offset w19, -85155; CHECK-GI-NEXT:    .cfi_offset w20, -165156; CHECK-GI-NEXT:    .cfi_offset w21, -245157; CHECK-GI-NEXT:    .cfi_offset w22, -325158; CHECK-GI-NEXT:    .cfi_offset w23, -405159; CHECK-GI-NEXT:    .cfi_offset w24, -485160; CHECK-GI-NEXT:    .cfi_offset w25, -565161; CHECK-GI-NEXT:    .cfi_offset w26, -645162; CHECK-GI-NEXT:    .cfi_offset w27, -725163; CHECK-GI-NEXT:    .cfi_offset w28, -805164; CHECK-GI-NEXT:    .cfi_offset w30, -885165; CHECK-GI-NEXT:    .cfi_offset w29, -965166; CHECK-GI-NEXT:    ldp q7, q16, [x1]5167; CHECK-GI-NEXT:    movi d1, #00000000000000005168; CHECK-GI-NEXT:    str w2, [sp, #12] // 4-byte Spill5169; CHECK-GI-NEXT:    movi d3, #00000000000000005170; CHECK-GI-NEXT:    movi d2, #00000000000000005171; CHECK-GI-NEXT:    movi d5, #00000000000000005172; CHECK-GI-NEXT:    movi d4, #00000000000000005173; CHECK-GI-NEXT:    movi d6, #00000000000000005174; CHECK-GI-NEXT:    mov b19, v7.b[3]5175; CHECK-GI-NEXT:    mov b23, v7.b[7]5176; CHECK-GI-NEXT:    mov b17, v7.b[1]5177; CHECK-GI-NEXT:    fmov w11, s75178; CHECK-GI-NEXT:    mov b18, v7.b[2]5179; CHECK-GI-NEXT:    mov b20, v7.b[4]5180; CHECK-GI-NEXT:    mov b21, v7.b[5]5181; CHECK-GI-NEXT:    mov b22, v7.b[6]5182; CHECK-GI-NEXT:    mov b24, v7.b[8]5183; CHECK-GI-NEXT:    mov b25, v7.b[9]5184; CHECK-GI-NEXT:    mov b26, v7.b[10]5185; CHECK-GI-NEXT:    mov b27, v7.b[11]5186; CHECK-GI-NEXT:    sxtb w11, w115187; CHECK-GI-NEXT:    mov b28, v7.b[12]5188; CHECK-GI-NEXT:    fmov w14, s195189; CHECK-GI-NEXT:    mov b19, v7.b[13]5190; CHECK-GI-NEXT:    mov b29, v7.b[14]5191; CHECK-GI-NEXT:    mov b7, v7.b[15]5192; CHECK-GI-NEXT:    fmov w7, s235193; CHECK-GI-NEXT:    mov b23, v16.b[6]5194; CHECK-GI-NEXT:    fmov w10, s175195; CHECK-GI-NEXT:    fmov w9, s185196; CHECK-GI-NEXT:    fmov w8, s245197; CHECK-GI-NEXT:    mov b30, v16.b[1]5198; CHECK-GI-NEXT:    fmov w16, s255199; CHECK-GI-NEXT:    fmov w12, s205200; CHECK-GI-NEXT:    fmov w24, s215201; CHECK-GI-NEXT:    sxtb w10, w105202; CHECK-GI-NEXT:    sxtb w7, w75203; CHECK-GI-NEXT:    fmov w22, s225204; CHECK-GI-NEXT:    stp s23, s7, [sp, #4] // 8-byte Folded Spill5205; CHECK-GI-NEXT:    sxtb w9, w95206; CHECK-GI-NEXT:    sxtb w8, w85207; CHECK-GI-NEXT:    fmov s7, w115208; CHECK-GI-NEXT:    mov b20, v16.b[2]5209; CHECK-GI-NEXT:    mov b17, v16.b[3]5210; CHECK-GI-NEXT:    mov b21, v16.b[4]5211; CHECK-GI-NEXT:    mov b18, v16.b[5]5212; CHECK-GI-NEXT:    fmov w27, s265213; CHECK-GI-NEXT:    fmov w25, s275214; CHECK-GI-NEXT:    mov b22, v16.b[7]5215; CHECK-GI-NEXT:    fmov w26, s285216; CHECK-GI-NEXT:    mov v7.h[1], w105217; CHECK-GI-NEXT:    sxtb w10, w165218; CHECK-GI-NEXT:    mov b25, v16.b[8]5219; CHECK-GI-NEXT:    fmov w23, s195220; CHECK-GI-NEXT:    mov b24, v16.b[9]5221; CHECK-GI-NEXT:    fmov w5, s295222; CHECK-GI-NEXT:    mov b26, v16.b[10]5223; CHECK-GI-NEXT:    mov b19, v16.b[11]5224; CHECK-GI-NEXT:    fmov w6, s305225; CHECK-GI-NEXT:    mov b27, v16.b[12]5226; CHECK-GI-NEXT:    mov b28, v16.b[13]5227; CHECK-GI-NEXT:    mov b29, v16.b[14]5228; CHECK-GI-NEXT:    sxtb w30, w235229; CHECK-GI-NEXT:    sxtb w5, w55230; CHECK-GI-NEXT:    mov v7.h[2], w95231; CHECK-GI-NEXT:    sxtb w9, w145232; CHECK-GI-NEXT:    fmov w20, s165233; CHECK-GI-NEXT:    mov b30, v16.b[15]5234; CHECK-GI-NEXT:    fmov s16, w85235; CHECK-GI-NEXT:    sxtb w8, w125236; CHECK-GI-NEXT:    fmov w15, s175237; CHECK-GI-NEXT:    fmov w11, s185238; CHECK-GI-NEXT:    ldp q18, q17, [x0]5239; CHECK-GI-NEXT:    mov v7.h[3], w95240; CHECK-GI-NEXT:    sxtb w9, w275241; CHECK-GI-NEXT:    fmov w18, s205242; CHECK-GI-NEXT:    sxtb w15, w155243; CHECK-GI-NEXT:    mov v16.h[1], w105244; CHECK-GI-NEXT:    sxtb w10, w255245; CHECK-GI-NEXT:    mov b20, v18.b[3]5246; CHECK-GI-NEXT:    fmov w2, s225247; CHECK-GI-NEXT:    mov b22, v18.b[1]5248; CHECK-GI-NEXT:    sxtb w18, w185249; CHECK-GI-NEXT:    fmov w13, s215250; CHECK-GI-NEXT:    mov b21, v18.b[2]5251; CHECK-GI-NEXT:    mov v7.h[4], w85252; CHECK-GI-NEXT:    fmov w3, s195253; CHECK-GI-NEXT:    mov b19, v18.b[6]5254; CHECK-GI-NEXT:    mov v16.h[2], w95255; CHECK-GI-NEXT:    sxtb w9, w245256; CHECK-GI-NEXT:    fmov w21, s255257; CHECK-GI-NEXT:    sxtb w13, w135258; CHECK-GI-NEXT:    fmov w28, s205259; CHECK-GI-NEXT:    mov b20, v18.b[11]5260; CHECK-GI-NEXT:    fmov w8, s225261; CHECK-GI-NEXT:    mov b25, v18.b[8]5262; CHECK-GI-NEXT:    fmov w29, s215263; CHECK-GI-NEXT:    mov v7.h[5], w95264; CHECK-GI-NEXT:    sxtb w9, w225265; CHECK-GI-NEXT:    fmov w19, s245266; CHECK-GI-NEXT:    mov v16.h[3], w105267; CHECK-GI-NEXT:    sxtb w10, w265268; CHECK-GI-NEXT:    fmov w26, s185269; CHECK-GI-NEXT:    sxtb w8, w85270; CHECK-GI-NEXT:    sxtb w29, w295271; CHECK-GI-NEXT:    mov b24, v18.b[4]5272; CHECK-GI-NEXT:    mov b23, v18.b[5]5273; CHECK-GI-NEXT:    fmov w17, s275274; CHECK-GI-NEXT:    mov b27, v18.b[9]5275; CHECK-GI-NEXT:    sxtb w23, w265276; CHECK-GI-NEXT:    mov v7.h[6], w95277; CHECK-GI-NEXT:    fmov w24, s195278; CHECK-GI-NEXT:    mov v16.h[4], w105279; CHECK-GI-NEXT:    mov b19, v18.b[14]5280; CHECK-GI-NEXT:    fmov w10, s255281; CHECK-GI-NEXT:    fmov w4, s265282; CHECK-GI-NEXT:    fmov w16, s285283; CHECK-GI-NEXT:    mov b26, v18.b[7]5284; CHECK-GI-NEXT:    mov b28, v18.b[10]5285; CHECK-GI-NEXT:    fmov w27, s245286; CHECK-GI-NEXT:    mov b24, v18.b[12]5287; CHECK-GI-NEXT:    sxtb w10, w105288; CHECK-GI-NEXT:    mov v7.h[7], w75289; CHECK-GI-NEXT:    fmov w7, s205290; CHECK-GI-NEXT:    sxtb w4, w45291; CHECK-GI-NEXT:    fmov s20, w235292; CHECK-GI-NEXT:    fmov w25, s235293; CHECK-GI-NEXT:    mov b23, v18.b[13]5294; CHECK-GI-NEXT:    mov b22, v18.b[15]5295; CHECK-GI-NEXT:    mov v16.h[5], w305296; CHECK-GI-NEXT:    sxtb w7, w75297; CHECK-GI-NEXT:    fmov w9, s275298; CHECK-GI-NEXT:    mov b21, v17.b[1]5299; CHECK-GI-NEXT:    mov v20.h[1], w85300; CHECK-GI-NEXT:    sxtb w8, w205301; CHECK-GI-NEXT:    sxtb w20, w65302; CHECK-GI-NEXT:    fmov w6, s195303; CHECK-GI-NEXT:    fmov w26, s285304; CHECK-GI-NEXT:    mov b28, v17.b[8]5305; CHECK-GI-NEXT:    fmov s18, w85306; CHECK-GI-NEXT:    sxtb w8, w215307; CHECK-GI-NEXT:    mov v16.h[6], w55308; CHECK-GI-NEXT:    fmov w5, s225309; CHECK-GI-NEXT:    fmov s22, w105310; CHECK-GI-NEXT:    sxtb w10, w275311; CHECK-GI-NEXT:    sxtb w26, w265312; CHECK-GI-NEXT:    mov v20.h[2], w295313; CHECK-GI-NEXT:    fmov s19, w85314; CHECK-GI-NEXT:    sxtb w8, w285315; CHECK-GI-NEXT:    sxtb w28, w195316; CHECK-GI-NEXT:    sxtb w19, w95317; CHECK-GI-NEXT:    fmov w27, s175318; CHECK-GI-NEXT:    mov b25, v17.b[2]5319; CHECK-GI-NEXT:    fmov w29, s215320; CHECK-GI-NEXT:    mov b21, v17.b[9]5321; CHECK-GI-NEXT:    mov v22.h[1], w195322; CHECK-GI-NEXT:    fmov w23, s235323; CHECK-GI-NEXT:    mov v20.h[3], w85324; CHECK-GI-NEXT:    mov b23, v17.b[6]5325; CHECK-GI-NEXT:    fmov w30, s245326; CHECK-GI-NEXT:    sxtb w27, w275327; CHECK-GI-NEXT:    mov b24, v17.b[5]5328; CHECK-GI-NEXT:    mov v18.h[1], w205329; CHECK-GI-NEXT:    fmov w21, s255330; CHECK-GI-NEXT:    mov b25, v17.b[10]5331; CHECK-GI-NEXT:    mov v19.h[1], w285332; CHECK-GI-NEXT:    sxtb w28, w295333; CHECK-GI-NEXT:    mov v22.h[2], w265334; CHECK-GI-NEXT:    fmov w26, s215335; CHECK-GI-NEXT:    mov v20.h[4], w105336; CHECK-GI-NEXT:    fmov w10, s285337; CHECK-GI-NEXT:    fmov s21, w275338; CHECK-GI-NEXT:    sxtb w21, w215339; CHECK-GI-NEXT:    mov b27, v17.b[3]5340; CHECK-GI-NEXT:    fmov w19, s235341; CHECK-GI-NEXT:    sxtb w26, w265342; CHECK-GI-NEXT:    fmov w22, s265343; CHECK-GI-NEXT:    mov b26, v17.b[4]5344; CHECK-GI-NEXT:    sxtb w10, w105345; CHECK-GI-NEXT:    mov v21.h[1], w285346; CHECK-GI-NEXT:    fmov w8, s245347; CHECK-GI-NEXT:    mov b24, v17.b[11]5348; CHECK-GI-NEXT:    fmov w27, s255349; CHECK-GI-NEXT:    mov v18.h[2], w185350; CHECK-GI-NEXT:    sxtb w18, w255351; CHECK-GI-NEXT:    fmov s23, w105352; CHECK-GI-NEXT:    fmov w20, s275353; CHECK-GI-NEXT:    sxtb w8, w85354; CHECK-GI-NEXT:    fmov w9, s265355; CHECK-GI-NEXT:    mov b26, v17.b[12]5356; CHECK-GI-NEXT:    sxtb w25, w275357; CHECK-GI-NEXT:    mov v20.h[5], w185358; CHECK-GI-NEXT:    sxtb w18, w35359; CHECK-GI-NEXT:    sxtb w3, w245360; CHECK-GI-NEXT:    mov v23.h[1], w265361; CHECK-GI-NEXT:    mov v21.h[2], w215362; CHECK-GI-NEXT:    sxtb w9, w95363; CHECK-GI-NEXT:    fmov w28, s245364; CHECK-GI-NEXT:    mov v22.h[3], w75365; CHECK-GI-NEXT:    sxtb w7, w205366; CHECK-GI-NEXT:    mov v19.h[2], w45367; CHECK-GI-NEXT:    sxtb w4, w305368; CHECK-GI-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload5369; CHECK-GI-NEXT:    mov v18.h[3], w155370; CHECK-GI-NEXT:    sxtb w20, w285371; CHECK-GI-NEXT:    sxtb w15, w175372; CHECK-GI-NEXT:    sxtb w17, w225373; CHECK-GI-NEXT:    ldp x22, x21, [sp, #80] // 16-byte Folded Reload5374; CHECK-GI-NEXT:    mov v23.h[2], w255375; CHECK-GI-NEXT:    mov v20.h[6], w35376; CHECK-GI-NEXT:    mov v21.h[3], w75377; CHECK-GI-NEXT:    fmov w10, s265378; CHECK-GI-NEXT:    mov v22.h[4], w45379; CHECK-GI-NEXT:    mov v19.h[3], w185380; CHECK-GI-NEXT:    sxtb w18, w235381; CHECK-GI-NEXT:    ldp x24, x23, [sp, #64] // 16-byte Folded Reload5382; CHECK-GI-NEXT:    mov b27, v17.b[13]5383; CHECK-GI-NEXT:    sxtb w10, w105384; CHECK-GI-NEXT:    mov v23.h[3], w205385; CHECK-GI-NEXT:    mov v18.h[4], w135386; CHECK-GI-NEXT:    sxtb w13, w65387; CHECK-GI-NEXT:    mov v20.h[7], w175388; CHECK-GI-NEXT:    mov v21.h[4], w95389; CHECK-GI-NEXT:    ldr w9, [sp, #8] // 4-byte Reload5390; CHECK-GI-NEXT:    mov v22.h[5], w185391; CHECK-GI-NEXT:    mov b25, v17.b[14]5392; CHECK-GI-NEXT:    fmov w26, s275393; CHECK-GI-NEXT:    mov v19.h[4], w155394; CHECK-GI-NEXT:    fmov w14, s295395; CHECK-GI-NEXT:    sxtb w9, w95396; CHECK-GI-NEXT:    mov v23.h[4], w105397; CHECK-GI-NEXT:    sxtb w10, w115398; CHECK-GI-NEXT:    sxtb w11, w165399; CHECK-GI-NEXT:    mov v21.h[5], w85400; CHECK-GI-NEXT:    ldr w8, [sp, #4] // 4-byte Reload5401; CHECK-GI-NEXT:    sxtb w15, w265402; CHECK-GI-NEXT:    ldp x26, x25, [sp, #48] // 16-byte Folded Reload5403; CHECK-GI-NEXT:    sxtb w8, w85404; CHECK-GI-NEXT:    mov v18.h[5], w105405; CHECK-GI-NEXT:    sxtb w10, w195406; CHECK-GI-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload5407; CHECK-GI-NEXT:    mul v20.8h, v7.8h, v20.8h5408; CHECK-GI-NEXT:    mov b7, v17.b[7]5409; CHECK-GI-NEXT:    mov v22.h[6], w135410; CHECK-GI-NEXT:    sxtb w13, w55411; CHECK-GI-NEXT:    fmov w27, s255412; CHECK-GI-NEXT:    mov v19.h[5], w115413; CHECK-GI-NEXT:    sxtb w11, w25414; CHECK-GI-NEXT:    mov b17, v17.b[15]5415; CHECK-GI-NEXT:    mov v18.h[6], w85416; CHECK-GI-NEXT:    mov v16.h[7], w95417; CHECK-GI-NEXT:    sxtb w9, w145418; CHECK-GI-NEXT:    mov v23.h[5], w155419; CHECK-GI-NEXT:    mov v21.h[6], w105420; CHECK-GI-NEXT:    sxtb w14, w275421; CHECK-GI-NEXT:    ldp x28, x27, [sp, #32] // 16-byte Folded Reload5422; CHECK-GI-NEXT:    fmov w8, s75423; CHECK-GI-NEXT:    mov v22.h[7], w135424; CHECK-GI-NEXT:    fmov w12, s305425; CHECK-GI-NEXT:    mov v19.h[6], w95426; CHECK-GI-NEXT:    fmov w9, s175427; CHECK-GI-NEXT:    smov w10, v20.h[0]5428; CHECK-GI-NEXT:    mov v23.h[6], w145429; CHECK-GI-NEXT:    mov v18.h[7], w115430; CHECK-GI-NEXT:    smov w13, v20.h[1]5431; CHECK-GI-NEXT:    sxtb w8, w85432; CHECK-GI-NEXT:    sxtb w12, w125433; CHECK-GI-NEXT:    smov w11, v20.h[4]5434; CHECK-GI-NEXT:    sxtb w9, w95435; CHECK-GI-NEXT:    mul v22.8h, v16.8h, v22.8h5436; CHECK-GI-NEXT:    smov w14, v20.h[3]5437; CHECK-GI-NEXT:    mov v21.h[7], w85438; CHECK-GI-NEXT:    ldrsb w8, [x0, #32]5439; CHECK-GI-NEXT:    mov v19.h[7], w125440; CHECK-GI-NEXT:    mov v23.h[7], w95441; CHECK-GI-NEXT:    ldrsb w9, [x1, #32]5442; CHECK-GI-NEXT:    fmov s7, w105443; CHECK-GI-NEXT:    smov w10, v20.h[2]5444; CHECK-GI-NEXT:    smov w12, v20.h[5]5445; CHECK-GI-NEXT:    fmov s16, w115446; CHECK-GI-NEXT:    mul w9, w9, w85447; CHECK-GI-NEXT:    smov w15, v22.h[4]5448; CHECK-GI-NEXT:    smov w17, v22.h[5]5449; CHECK-GI-NEXT:    mul v24.8h, v18.8h, v21.8h5450; CHECK-GI-NEXT:    mov v7.s[1], w135451; CHECK-GI-NEXT:    smov w13, v22.h[0]5452; CHECK-GI-NEXT:    mul v18.8h, v19.8h, v23.8h5453; CHECK-GI-NEXT:    smov w16, v22.h[1]5454; CHECK-GI-NEXT:    smov w8, v20.h[7]5455; CHECK-GI-NEXT:    sxth w9, w95456; CHECK-GI-NEXT:    mov v16.s[1], w125457; CHECK-GI-NEXT:    movi d0, #00000000000000005458; CHECK-GI-NEXT:    fmov s19, w155459; CHECK-GI-NEXT:    smov w15, v22.h[6]5460; CHECK-GI-NEXT:    mov v1.s[1], wzr5461; CHECK-GI-NEXT:    smov w11, v24.h[0]5462; CHECK-GI-NEXT:    mov v7.s[2], w105463; CHECK-GI-NEXT:    smov w10, v20.h[6]5464; CHECK-GI-NEXT:    smov w12, v24.h[1]5465; CHECK-GI-NEXT:    smov w0, v18.h[4]5466; CHECK-GI-NEXT:    fmov s17, w135467; CHECK-GI-NEXT:    mov v19.s[1], w175468; CHECK-GI-NEXT:    smov w17, v18.h[0]5469; CHECK-GI-NEXT:    smov w18, v18.h[1]5470; CHECK-GI-NEXT:    smov w13, v22.h[2]5471; CHECK-GI-NEXT:    mov v3.s[1], wzr5472; CHECK-GI-NEXT:    mov v2.s[1], wzr5473; CHECK-GI-NEXT:    fmov s20, w115474; CHECK-GI-NEXT:    smov w11, v24.h[4]5475; CHECK-GI-NEXT:    mov v7.s[3], w145476; CHECK-GI-NEXT:    smov w14, v24.h[5]5477; CHECK-GI-NEXT:    mov v17.s[1], w165478; CHECK-GI-NEXT:    smov w16, v24.h[2]5479; CHECK-GI-NEXT:    mov v19.s[2], w155480; CHECK-GI-NEXT:    smov w15, v18.h[5]5481; CHECK-GI-NEXT:    fmov s23, w05482; CHECK-GI-NEXT:    mov v20.s[1], w125483; CHECK-GI-NEXT:    mov v16.s[2], w105484; CHECK-GI-NEXT:    smov w10, v22.h[3]5485; CHECK-GI-NEXT:    fmov s21, w115486; CHECK-GI-NEXT:    smov w11, v22.h[7]5487; CHECK-GI-NEXT:    fmov s22, w175488; CHECK-GI-NEXT:    mov v5.s[1], wzr5489; CHECK-GI-NEXT:    mov v4.s[1], wzr5490; CHECK-GI-NEXT:    mov v6.s[1], wzr5491; CHECK-GI-NEXT:    mov v23.s[1], w155492; CHECK-GI-NEXT:    smov w15, v18.h[6]5493; CHECK-GI-NEXT:    mov v0.s[1], wzr5494; CHECK-GI-NEXT:    mov v21.s[1], w145495; CHECK-GI-NEXT:    smov w14, v24.h[6]5496; CHECK-GI-NEXT:    mov v20.s[2], w165497; CHECK-GI-NEXT:    mov v22.s[1], w185498; CHECK-GI-NEXT:    smov w16, v18.h[2]5499; CHECK-GI-NEXT:    mov v1.s[2], wzr5500; CHECK-GI-NEXT:    mov v3.s[2], wzr5501; CHECK-GI-NEXT:    mov v2.s[2], wzr5502; CHECK-GI-NEXT:    mov v5.s[2], wzr5503; CHECK-GI-NEXT:    mov v4.s[2], wzr5504; CHECK-GI-NEXT:    mov v6.s[2], wzr5505; CHECK-GI-NEXT:    mov v23.s[2], w155506; CHECK-GI-NEXT:    mov v21.s[2], w145507; CHECK-GI-NEXT:    smov w14, v18.h[3]5508; CHECK-GI-NEXT:    smov w15, v18.h[7]5509; CHECK-GI-NEXT:    fmov s18, w95510; CHECK-GI-NEXT:    ldr w9, [sp, #12] // 4-byte Reload5511; CHECK-GI-NEXT:    mov v17.s[2], w135512; CHECK-GI-NEXT:    smov w12, v24.h[3]5513; CHECK-GI-NEXT:    smov w13, v24.h[7]5514; CHECK-GI-NEXT:    mov v22.s[2], w165515; CHECK-GI-NEXT:    mov v0.s[2], wzr5516; CHECK-GI-NEXT:    mov v1.s[3], wzr5517; CHECK-GI-NEXT:    mov v3.s[3], wzr5518; CHECK-GI-NEXT:    mov v2.s[3], wzr5519; CHECK-GI-NEXT:    mov v5.s[3], wzr5520; CHECK-GI-NEXT:    mov v4.s[3], wzr5521; CHECK-GI-NEXT:    mov v6.s[3], wzr5522; CHECK-GI-NEXT:    mov v18.s[1], wzr5523; CHECK-GI-NEXT:    mov v16.s[3], w85524; CHECK-GI-NEXT:    mov v17.s[3], w105525; CHECK-GI-NEXT:    mov v19.s[3], w115526; CHECK-GI-NEXT:    mov v20.s[3], w125527; CHECK-GI-NEXT:    mov v21.s[3], w135528; CHECK-GI-NEXT:    mov v22.s[3], w145529; CHECK-GI-NEXT:    mov v23.s[3], w155530; CHECK-GI-NEXT:    mov v0.s[3], wzr5531; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s5532; CHECK-GI-NEXT:    add v2.4s, v2.4s, v5.4s5533; CHECK-GI-NEXT:    add v3.4s, v4.4s, v6.4s5534; CHECK-GI-NEXT:    mov v18.s[2], wzr5535; CHECK-GI-NEXT:    add v4.4s, v7.4s, v16.4s5536; CHECK-GI-NEXT:    add v5.4s, v17.4s, v19.4s5537; CHECK-GI-NEXT:    add v6.4s, v20.4s, v21.4s5538; CHECK-GI-NEXT:    add v7.4s, v22.4s, v23.4s5539; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s5540; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s5541; CHECK-GI-NEXT:    mov v18.s[3], wzr5542; CHECK-GI-NEXT:    add v2.4s, v4.4s, v5.4s5543; CHECK-GI-NEXT:    add v3.4s, v6.4s, v7.4s5544; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s5545; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s5546; CHECK-GI-NEXT:    add v0.4s, v18.4s, v0.4s5547; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s5548; CHECK-GI-NEXT:    addv s0, v0.4s5549; CHECK-GI-NEXT:    fmov w8, s05550; CHECK-GI-NEXT:    add w0, w8, w95551; CHECK-GI-NEXT:    add sp, sp, #1125552; CHECK-GI-NEXT:    ret5553entry:5554  %0 = load <33 x i8>, ptr %a5555  %1 = sext <33 x i8> %0 to <33 x i32>5556  %2 = load <33 x i8>, ptr %b5557  %3 = sext <33 x i8> %2 to <33 x i32>5558  %4 = mul nsw <33 x i32> %3, %15559  %5 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %4)5560  %op.extra = add nsw i32 %5, %sum5561  ret i32 %op.extra5562}5563 5564define i32 @test_sdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33 x i8> %d) {5565; CHECK-SD-LABEL: test_sdot_v33i8_double:5566; CHECK-SD:       // %bb.0: // %entry5567; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill5568; CHECK-SD-NEXT:    .cfi_def_cfa_offset 165569; CHECK-SD-NEXT:    .cfi_offset w29, -165570; CHECK-SD-NEXT:    ldr b0, [sp, #344]5571; CHECK-SD-NEXT:    add x8, sp, #3525572; CHECK-SD-NEXT:    ldr b1, [sp, #80]5573; CHECK-SD-NEXT:    ldr b2, [sp, #216]5574; CHECK-SD-NEXT:    add x9, sp, #965575; CHECK-SD-NEXT:    add x10, sp, #1045576; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]5577; CHECK-SD-NEXT:    add x8, sp, #885578; CHECK-SD-NEXT:    ldr b4, [sp, #408]5579; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x8]5580; CHECK-SD-NEXT:    add x8, sp, #3605581; CHECK-SD-NEXT:    add x12, sp, #2485582; CHECK-SD-NEXT:    add x13, sp, #4325583; CHECK-SD-NEXT:    add x11, sp, #3845584; CHECK-SD-NEXT:    ldr b5, [sp, #144]5585; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]5586; CHECK-SD-NEXT:    add x8, sp, #2245587; CHECK-SD-NEXT:    ldr b6, [sp, #280]5588; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x8]5589; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x9]5590; CHECK-SD-NEXT:    add x8, sp, #3685591; CHECK-SD-NEXT:    add x9, sp, #2325592; CHECK-SD-NEXT:    ldr b16, [sp, #744]5593; CHECK-SD-NEXT:    ldr b17, [sp, #480]5594; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x8]5595; CHECK-SD-NEXT:    add x8, sp, #3765596; CHECK-SD-NEXT:    ldr b18, [sp, #936]5597; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x9]5598; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x10]5599; CHECK-SD-NEXT:    add x9, sp, #2405600; CHECK-SD-NEXT:    add x10, sp, #3925601; CHECK-SD-NEXT:    ldr b19, [sp, #672]5602; CHECK-SD-NEXT:    ldr b7, [sp, #16]5603; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x8]5604; CHECK-SD-NEXT:    add x8, sp, #1125605; CHECK-SD-NEXT:    ldr b21, [sp, #1000]5606; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x9]5607; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x8]5608; CHECK-SD-NEXT:    add x8, sp, #4165609; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x8]5610; CHECK-SD-NEXT:    add x8, sp, #1205611; CHECK-SD-NEXT:    add x9, sp, #4005612; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x11]5613; CHECK-SD-NEXT:    add x11, sp, #1285614; CHECK-SD-NEXT:    ldr b22, [sp, #736]5615; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x12]5616; CHECK-SD-NEXT:    add x12, sp, #4245617; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x8]5618; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x12]5619; CHECK-SD-NEXT:    add x12, sp, #1525620; CHECK-SD-NEXT:    add x8, sp, #1365621; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x12]5622; CHECK-SD-NEXT:    add x12, sp, #4405623; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x10]5624; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x11]5625; CHECK-SD-NEXT:    add x11, sp, #2885626; CHECK-SD-NEXT:    add x10, sp, #2565627; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x13]5628; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x11]5629; CHECK-SD-NEXT:    add x11, sp, #2965630; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x9]5631; CHECK-SD-NEXT:    add x9, sp, #1605632; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x10]5633; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x9]5634; CHECK-SD-NEXT:    add x10, sp, #1685635; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x8]5636; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x12]5637; CHECK-SD-NEXT:    add x12, sp, #4485638; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x11]5639; CHECK-SD-NEXT:    add x11, sp, #3045640; CHECK-SD-NEXT:    add x8, sp, #4645641; CHECK-SD-NEXT:    add x13, sp, #7685642; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x10]5643; CHECK-SD-NEXT:    add x10, sp, #1765644; CHECK-SD-NEXT:    add x9, sp, #2645645; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x12]5646; CHECK-SD-NEXT:    add x12, sp, #4565647; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x11]5648; CHECK-SD-NEXT:    add x11, sp, #7605649; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x9]5650; CHECK-SD-NEXT:    add x9, sp, #2725651; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x10]5652; CHECK-SD-NEXT:    add x10, sp, #3125653; CHECK-SD-NEXT:    fmov s3, w05654; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x12]5655; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x10]5656; CHECK-SD-NEXT:    add x10, sp, #3205657; CHECK-SD-NEXT:    add x12, sp, #6805658; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x9]5659; CHECK-SD-NEXT:    add x9, sp, #1845660; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x12]5661; CHECK-SD-NEXT:    add x12, sp, #7765662; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x9]5663; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x8]5664; CHECK-SD-NEXT:    add x8, sp, #7525665; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x10]5666; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x8]5667; CHECK-SD-NEXT:    add x10, sp, #245668; CHECK-SD-NEXT:    smull v22.8h, v22.8b, v21.8b5669; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x10]5670; CHECK-SD-NEXT:    add x10, sp, #4965671; CHECK-SD-NEXT:    mov v3.b[1], w15672; CHECK-SD-NEXT:    add x9, sp, #1925673; CHECK-SD-NEXT:    ldr b20, [sp, #472]5674; CHECK-SD-NEXT:    ldr b23, [sp, #208]5675; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x11]5676; CHECK-SD-NEXT:    add x11, sp, #4885677; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x9]5678; CHECK-SD-NEXT:    ld1 { v17.b }[1], [x11]5679; CHECK-SD-NEXT:    add x11, sp, #9445680; CHECK-SD-NEXT:    add x9, sp, #3285681; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x11]5682; CHECK-SD-NEXT:    add x11, sp, #6885683; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x9]5684; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x13]5685; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x11]5686; CHECK-SD-NEXT:    add x11, sp, #5045687; CHECK-SD-NEXT:    ld1 { v17.b }[2], [x10]5688; CHECK-SD-NEXT:    add x10, sp, #9525689; CHECK-SD-NEXT:    add x13, sp, #7845690; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x10]5691; CHECK-SD-NEXT:    add x10, sp, #325692; CHECK-SD-NEXT:    add x9, sp, #405693; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x12]5694; CHECK-SD-NEXT:    add x12, sp, #6965695; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x10]5696; CHECK-SD-NEXT:    ld1 { v17.b }[3], [x11]5697; CHECK-SD-NEXT:    add x11, sp, #9605698; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x12]5699; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x11]5700; CHECK-SD-NEXT:    add x10, sp, #5125701; CHECK-SD-NEXT:    add x11, sp, #7045702; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x13]5703; CHECK-SD-NEXT:    add x12, sp, #7925704; CHECK-SD-NEXT:    sshll v24.4s, v22.4h, #05705; CHECK-SD-NEXT:    ld1 { v17.b }[4], [x10]5706; CHECK-SD-NEXT:    add x10, sp, #9685707; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x11]5708; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x10]5709; CHECK-SD-NEXT:    add x10, sp, #5205710; CHECK-SD-NEXT:    add x11, sp, #9765711; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x12]5712; CHECK-SD-NEXT:    add x12, sp, #7125713; CHECK-SD-NEXT:    smull v20.8h, v23.8b, v20.8b5714; CHECK-SD-NEXT:    ld1 { v17.b }[5], [x10]5715; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x12]5716; CHECK-SD-NEXT:    add x12, sp, #7205717; CHECK-SD-NEXT:    ld1 { v18.b }[5], [x11]5718; CHECK-SD-NEXT:    add x11, sp, #5285719; CHECK-SD-NEXT:    add x10, sp, #8005720; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x10]5721; CHECK-SD-NEXT:    add x10, sp, #5365722; CHECK-SD-NEXT:    ldr b22, [sp, #872]5723; CHECK-SD-NEXT:    ld1 { v17.b }[6], [x11]5724; CHECK-SD-NEXT:    add x11, sp, #9845725; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x12]5726; CHECK-SD-NEXT:    ld1 { v18.b }[6], [x11]5727; CHECK-SD-NEXT:    add x11, sp, #9925728; CHECK-SD-NEXT:    add x12, sp, #7285729; CHECK-SD-NEXT:    ldr b23, [sp, #608]5730; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]5731; CHECK-SD-NEXT:    add x9, sp, #8805732; CHECK-SD-NEXT:    ld1 { v17.b }[7], [x10]5733; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x12]5734; CHECK-SD-NEXT:    add x10, sp, #8165735; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x11]5736; CHECK-SD-NEXT:    add x11, sp, #5525737; CHECK-SD-NEXT:    add x12, sp, #6165738; CHECK-SD-NEXT:    mov v3.b[2], w25739; CHECK-SD-NEXT:    ld1 { v22.b }[1], [x9]5740; CHECK-SD-NEXT:    ld1 { v23.b }[1], [x12]5741; CHECK-SD-NEXT:    smull v16.8h, v17.8b, v16.8b5742; CHECK-SD-NEXT:    add x12, sp, #5605743; CHECK-SD-NEXT:    add x9, sp, #8885744; CHECK-SD-NEXT:    smull v17.8h, v19.8b, v18.8b5745; CHECK-SD-NEXT:    ldr b18, [sp, #808]5746; CHECK-SD-NEXT:    ldr b19, [sp, #544]5747; CHECK-SD-NEXT:    add x13, sp, #6245748; CHECK-SD-NEXT:    ld1 { v22.b }[2], [x9]5749; CHECK-SD-NEXT:    add x9, sp, #8965750; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x10]5751; CHECK-SD-NEXT:    ld1 { v19.b }[1], [x11]5752; CHECK-SD-NEXT:    add x11, sp, #8245753; CHECK-SD-NEXT:    add x10, sp, #485754; CHECK-SD-NEXT:    ld1 { v23.b }[2], [x13]5755; CHECK-SD-NEXT:    mov v3.b[3], w35756; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]5757; CHECK-SD-NEXT:    add x10, sp, #8325758; CHECK-SD-NEXT:    ld1 { v22.b }[3], [x9]5759; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x11]5760; CHECK-SD-NEXT:    ld1 { v19.b }[2], [x12]5761; CHECK-SD-NEXT:    add x11, sp, #5685762; CHECK-SD-NEXT:    add x12, sp, #6325763; CHECK-SD-NEXT:    add x9, sp, #9045764; CHECK-SD-NEXT:    add x13, sp, #6405765; CHECK-SD-NEXT:    ld1 { v23.b }[3], [x12]5766; CHECK-SD-NEXT:    add x12, sp, #5765767; CHECK-SD-NEXT:    mov v3.b[4], w45768; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x10]5769; CHECK-SD-NEXT:    ld1 { v19.b }[3], [x11]5770; CHECK-SD-NEXT:    add x11, sp, #8405771; CHECK-SD-NEXT:    add x10, sp, #565772; CHECK-SD-NEXT:    ld1 { v22.b }[4], [x9]5773; CHECK-SD-NEXT:    add x9, sp, #9125774; CHECK-SD-NEXT:    ld1 { v23.b }[4], [x13]5775; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x10]5776; CHECK-SD-NEXT:    add x10, sp, #8485777; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x11]5778; CHECK-SD-NEXT:    ld1 { v19.b }[4], [x12]5779; CHECK-SD-NEXT:    add x11, sp, #5845780; CHECK-SD-NEXT:    add x12, sp, #6485781; CHECK-SD-NEXT:    mov v3.b[5], w55782; CHECK-SD-NEXT:    ld1 { v22.b }[5], [x9]5783; CHECK-SD-NEXT:    ld1 { v23.b }[5], [x12]5784; CHECK-SD-NEXT:    add x12, sp, #5925785; CHECK-SD-NEXT:    movi v21.2d, #00000000000000005786; CHECK-SD-NEXT:    ld1 { v18.b }[5], [x10]5787; CHECK-SD-NEXT:    ld1 { v19.b }[5], [x11]5788; CHECK-SD-NEXT:    add x11, sp, #8565789; CHECK-SD-NEXT:    add x9, sp, #9205790; CHECK-SD-NEXT:    add x13, sp, #6565791; CHECK-SD-NEXT:    add x10, sp, #645792; CHECK-SD-NEXT:    ld1 { v22.b }[6], [x9]5793; CHECK-SD-NEXT:    ld1 { v23.b }[6], [x13]5794; CHECK-SD-NEXT:    mov v3.b[6], w65795; CHECK-SD-NEXT:    ld1 { v18.b }[6], [x11]5796; CHECK-SD-NEXT:    ld1 { v19.b }[6], [x12]5797; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x10]5798; CHECK-SD-NEXT:    add x10, sp, #8645799; CHECK-SD-NEXT:    add x11, sp, #6005800; CHECK-SD-NEXT:    add x9, sp, #9285801; CHECK-SD-NEXT:    add x12, sp, #6645802; CHECK-SD-NEXT:    mov v21.s[0], v24.s[0]5803; CHECK-SD-NEXT:    ld1 { v22.b }[7], [x9]5804; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x10]5805; CHECK-SD-NEXT:    ld1 { v19.b }[7], [x11]5806; CHECK-SD-NEXT:    ld1 { v23.b }[7], [x12]5807; CHECK-SD-NEXT:    add x8, sp, #2005808; CHECK-SD-NEXT:    mov v3.b[7], w75809; CHECK-SD-NEXT:    add x10, sp, #3365810; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x8]5811; CHECK-SD-NEXT:    add x8, sp, #725812; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x10]5813; CHECK-SD-NEXT:    smull v18.8h, v19.8b, v18.8b5814; CHECK-SD-NEXT:    movi v19.2d, #00000000000000005815; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x8]5816; CHECK-SD-NEXT:    smull v22.8h, v23.8b, v22.8b5817; CHECK-SD-NEXT:    sshll v20.4s, v20.4h, #05818; CHECK-SD-NEXT:    smull v0.8h, v1.8b, v0.8b5819; CHECK-SD-NEXT:    saddw v1.4s, v21.4s, v16.4h5820; CHECK-SD-NEXT:    smull v2.8h, v3.8b, v2.8b5821; CHECK-SD-NEXT:    smull v3.8h, v5.8b, v4.8b5822; CHECK-SD-NEXT:    smull v4.8h, v7.8b, v6.8b5823; CHECK-SD-NEXT:    mov v19.s[0], v20.s[0]5824; CHECK-SD-NEXT:    saddl2 v5.4s, v18.8h, v17.8h5825; CHECK-SD-NEXT:    saddl v7.4s, v18.4h, v17.4h5826; CHECK-SD-NEXT:    saddl2 v6.4s, v16.8h, v22.8h5827; CHECK-SD-NEXT:    saddw v1.4s, v1.4s, v22.4h5828; CHECK-SD-NEXT:    saddl2 v17.4s, v2.8h, v0.8h5829; CHECK-SD-NEXT:    saddl2 v16.4s, v4.8h, v3.8h5830; CHECK-SD-NEXT:    saddl v3.4s, v4.4h, v3.4h5831; CHECK-SD-NEXT:    saddw v2.4s, v19.4s, v2.4h5832; CHECK-SD-NEXT:    add v5.4s, v6.4s, v5.4s5833; CHECK-SD-NEXT:    add v1.4s, v1.4s, v7.4s5834; CHECK-SD-NEXT:    add v6.4s, v17.4s, v16.4s5835; CHECK-SD-NEXT:    saddw v0.4s, v2.4s, v0.4h5836; CHECK-SD-NEXT:    add v1.4s, v1.4s, v5.4s5837; CHECK-SD-NEXT:    add v0.4s, v0.4s, v3.4s5838; CHECK-SD-NEXT:    add v1.4s, v6.4s, v1.4s5839; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s5840; CHECK-SD-NEXT:    addv s0, v0.4s5841; CHECK-SD-NEXT:    fmov w0, s05842; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload5843; CHECK-SD-NEXT:    ret5844;5845; CHECK-GI-LABEL: test_sdot_v33i8_double:5846; CHECK-GI:       // %bb.0: // %entry5847; CHECK-GI-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill5848; CHECK-GI-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill5849; CHECK-GI-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill5850; CHECK-GI-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill5851; CHECK-GI-NEXT:    str x29, [sp, #64] // 8-byte Spill5852; CHECK-GI-NEXT:    .cfi_def_cfa_offset 805853; CHECK-GI-NEXT:    .cfi_offset w29, -165854; CHECK-GI-NEXT:    .cfi_offset b8, -245855; CHECK-GI-NEXT:    .cfi_offset b9, -325856; CHECK-GI-NEXT:    .cfi_offset b10, -405857; CHECK-GI-NEXT:    .cfi_offset b11, -485858; CHECK-GI-NEXT:    .cfi_offset b12, -565859; CHECK-GI-NEXT:    .cfi_offset b13, -645860; CHECK-GI-NEXT:    .cfi_offset b14, -725861; CHECK-GI-NEXT:    .cfi_offset b15, -805862; CHECK-GI-NEXT:    lsl w8, w0, #85863; CHECK-GI-NEXT:    ldr w10, [sp, #80]5864; CHECK-GI-NEXT:    lsl w11, w1, #85865; CHECK-GI-NEXT:    ldr w9, [sp, #88]5866; CHECK-GI-NEXT:    ldr w13, [sp, #128]5867; CHECK-GI-NEXT:    ldr w14, [sp, #136]5868; CHECK-GI-NEXT:    sbfx w12, w8, #8, #85869; CHECK-GI-NEXT:    lsl w10, w10, #85870; CHECK-GI-NEXT:    sbfx w8, w11, #8, #85871; CHECK-GI-NEXT:    lsl w9, w9, #85872; CHECK-GI-NEXT:    lsl w11, w2, #85873; CHECK-GI-NEXT:    lsl w13, w13, #85874; CHECK-GI-NEXT:    fmov s22, w125875; CHECK-GI-NEXT:    sbfx w10, w10, #8, #85876; CHECK-GI-NEXT:    ldr w12, [sp, #152]5877; CHECK-GI-NEXT:    sbfx w9, w9, #8, #85878; CHECK-GI-NEXT:    lsl w16, w7, #85879; CHECK-GI-NEXT:    lsl w14, w14, #85880; CHECK-GI-NEXT:    fmov s23, w105881; CHECK-GI-NEXT:    sbfx w10, w11, #8, #85882; CHECK-GI-NEXT:    lsl w11, w3, #85883; CHECK-GI-NEXT:    mov v22.h[1], w85884; CHECK-GI-NEXT:    ldr w8, [sp, #96]5885; CHECK-GI-NEXT:    lsl w12, w12, #85886; CHECK-GI-NEXT:    sbfx w13, w13, #8, #85887; CHECK-GI-NEXT:    sbfx w16, w16, #8, #85888; CHECK-GI-NEXT:    ldr w15, [sp, #176]5889; CHECK-GI-NEXT:    lsl w8, w8, #85890; CHECK-GI-NEXT:    mov v23.h[1], w95891; CHECK-GI-NEXT:    ldr w9, [sp, #104]5892; CHECK-GI-NEXT:    sbfx w12, w12, #8, #85893; CHECK-GI-NEXT:    lsl w15, w15, #85894; CHECK-GI-NEXT:    ldr w17, [sp, #224]5895; CHECK-GI-NEXT:    mov v22.h[2], w105896; CHECK-GI-NEXT:    sbfx w8, w8, #8, #85897; CHECK-GI-NEXT:    sbfx w10, w11, #8, #85898; CHECK-GI-NEXT:    lsl w9, w9, #85899; CHECK-GI-NEXT:    lsl w11, w4, #85900; CHECK-GI-NEXT:    sbfx w15, w15, #8, #85901; CHECK-GI-NEXT:    mov v23.h[2], w85902; CHECK-GI-NEXT:    ldr w8, [sp, #112]5903; CHECK-GI-NEXT:    movi d19, #00000000000000005904; CHECK-GI-NEXT:    sbfx w9, w9, #8, #85905; CHECK-GI-NEXT:    sbfx w11, w11, #8, #85906; CHECK-GI-NEXT:    movi d21, #00000000000000005907; CHECK-GI-NEXT:    mov v22.h[3], w105908; CHECK-GI-NEXT:    ldr w10, [sp, #144]5909; CHECK-GI-NEXT:    lsl w8, w8, #85910; CHECK-GI-NEXT:    movi d16, #00000000000000005911; CHECK-GI-NEXT:    movi d18, #00000000000000005912; CHECK-GI-NEXT:    movi d17, #00000000000000005913; CHECK-GI-NEXT:    lsl w10, w10, #85914; CHECK-GI-NEXT:    mov v23.h[3], w95915; CHECK-GI-NEXT:    sbfx w8, w8, #8, #85916; CHECK-GI-NEXT:    ldr w9, [sp, #120]5917; CHECK-GI-NEXT:    movi d20, #00000000000000005918; CHECK-GI-NEXT:    movi d6, #00000000000000005919; CHECK-GI-NEXT:    sbfx w10, w10, #8, #85920; CHECK-GI-NEXT:    mov v22.h[4], w115921; CHECK-GI-NEXT:    lsl w11, w5, #85922; CHECK-GI-NEXT:    lsl w9, w9, #85923; CHECK-GI-NEXT:    movi d7, #00000000000000005924; CHECK-GI-NEXT:    movi d2, #00000000000000005925; CHECK-GI-NEXT:    fmov s24, w105926; CHECK-GI-NEXT:    mov v23.h[4], w85927; CHECK-GI-NEXT:    ldr w8, [sp, #160]5928; CHECK-GI-NEXT:    sbfx w11, w11, #8, #85929; CHECK-GI-NEXT:    ldr w10, [sp, #168]5930; CHECK-GI-NEXT:    sbfx w9, w9, #8, #85931; CHECK-GI-NEXT:    lsl w8, w8, #85932; CHECK-GI-NEXT:    movi d4, #00000000000000005933; CHECK-GI-NEXT:    movi d3, #00000000000000005934; CHECK-GI-NEXT:    mov v24.h[1], w125935; CHECK-GI-NEXT:    lsl w12, w6, #85936; CHECK-GI-NEXT:    mov v22.h[5], w115937; CHECK-GI-NEXT:    sbfx w8, w8, #8, #85938; CHECK-GI-NEXT:    lsl w10, w10, #85939; CHECK-GI-NEXT:    mov v23.h[5], w95940; CHECK-GI-NEXT:    sbfx w12, w12, #8, #85941; CHECK-GI-NEXT:    ldr w11, [sp, #184]5942; CHECK-GI-NEXT:    ldr w9, [sp, #192]5943; CHECK-GI-NEXT:    sbfx w10, w10, #8, #85944; CHECK-GI-NEXT:    movi d5, #00000000000000005945; CHECK-GI-NEXT:    movi d1, #00000000000000005946; CHECK-GI-NEXT:    mov v24.h[2], w85947; CHECK-GI-NEXT:    mov v22.h[6], w125948; CHECK-GI-NEXT:    ldr w12, [sp, #208]5949; CHECK-GI-NEXT:    mov v23.h[6], w135950; CHECK-GI-NEXT:    ldr w13, [sp, #216]5951; CHECK-GI-NEXT:    lsl w9, w9, #85952; CHECK-GI-NEXT:    lsl w12, w12, #85953; CHECK-GI-NEXT:    ldr w8, [sp, #200]5954; CHECK-GI-NEXT:    movi d0, #00000000000000005955; CHECK-GI-NEXT:    lsl w13, w13, #85956; CHECK-GI-NEXT:    sbfx w9, w9, #8, #85957; CHECK-GI-NEXT:    mov v19.s[1], wzr5958; CHECK-GI-NEXT:    mov v24.h[3], w105959; CHECK-GI-NEXT:    sbfx w10, w14, #8, #85960; CHECK-GI-NEXT:    ldr w14, [sp, #280]5961; CHECK-GI-NEXT:    mov v22.h[7], w165962; CHECK-GI-NEXT:    ldr w16, [sp, #288]5963; CHECK-GI-NEXT:    sbfx w12, w12, #8, #85964; CHECK-GI-NEXT:    lsl w14, w14, #85965; CHECK-GI-NEXT:    sbfx w13, w13, #8, #85966; CHECK-GI-NEXT:    mov v23.h[7], w105967; CHECK-GI-NEXT:    lsl w18, w16, #85968; CHECK-GI-NEXT:    fmov s27, w125969; CHECK-GI-NEXT:    ldr w10, [sp, #232]5970; CHECK-GI-NEXT:    sbfx w16, w14, #8, #85971; CHECK-GI-NEXT:    mov v24.h[4], w155972; CHECK-GI-NEXT:    lsl w15, w11, #85973; CHECK-GI-NEXT:    sbfx w14, w18, #8, #85974; CHECK-GI-NEXT:    ldr w11, [sp, #296]5975; CHECK-GI-NEXT:    lsl w10, w10, #85976; CHECK-GI-NEXT:    fmov s25, w165977; CHECK-GI-NEXT:    ldr w16, [sp, #344]5978; CHECK-GI-NEXT:    mov v27.h[1], w135979; CHECK-GI-NEXT:    lsl w13, w17, #85980; CHECK-GI-NEXT:    lsl w11, w11, #85981; CHECK-GI-NEXT:    sbfx w15, w15, #8, #85982; CHECK-GI-NEXT:    lsl w16, w16, #85983; CHECK-GI-NEXT:    ldr w12, [sp, #240]5984; CHECK-GI-NEXT:    sbfx w17, w10, #8, #85985; CHECK-GI-NEXT:    mov v25.h[1], w145986; CHECK-GI-NEXT:    ldr w14, [sp, #352]5987; CHECK-GI-NEXT:    sbfx w13, w13, #8, #85988; CHECK-GI-NEXT:    sbfx w16, w16, #8, #85989; CHECK-GI-NEXT:    sbfx w11, w11, #8, #85990; CHECK-GI-NEXT:    mov v24.h[5], w155991; CHECK-GI-NEXT:    mov v27.h[2], w135992; CHECK-GI-NEXT:    lsl w13, w14, #85993; CHECK-GI-NEXT:    ldr w14, [sp, #304]5994; CHECK-GI-NEXT:    fmov s26, w165995; CHECK-GI-NEXT:    lsl w12, w12, #85996; CHECK-GI-NEXT:    ldr w15, [sp, #248]5997; CHECK-GI-NEXT:    mov v25.h[2], w115998; CHECK-GI-NEXT:    ldr w11, [sp, #360]5999; CHECK-GI-NEXT:    sbfx w13, w13, #8, #86000; CHECK-GI-NEXT:    lsl w14, w14, #86001; CHECK-GI-NEXT:    sbfx w12, w12, #8, #86002; CHECK-GI-NEXT:    mov v24.h[6], w96003; CHECK-GI-NEXT:    lsl w16, w11, #86004; CHECK-GI-NEXT:    mov v26.h[1], w136005; CHECK-GI-NEXT:    mov v27.h[3], w176006; CHECK-GI-NEXT:    sbfx w13, w14, #8, #86007; CHECK-GI-NEXT:    ldr w14, [sp, #312]6008; CHECK-GI-NEXT:    ldr w17, [sp, #328]6009; CHECK-GI-NEXT:    sbfx w16, w16, #8, #86010; CHECK-GI-NEXT:    ldr w10, [sp, #256]6011; CHECK-GI-NEXT:    ldr w11, [sp, #264]6012; CHECK-GI-NEXT:    mov v25.h[3], w136013; CHECK-GI-NEXT:    ldr w13, [sp, #368]6014; CHECK-GI-NEXT:    lsl w14, w14, #86015; CHECK-GI-NEXT:    mov v26.h[2], w166016; CHECK-GI-NEXT:    ldr w16, [sp, #320]6017; CHECK-GI-NEXT:    mov v27.h[4], w126018; CHECK-GI-NEXT:    lsl w13, w13, #86019; CHECK-GI-NEXT:    sbfx w9, w14, #8, #86020; CHECK-GI-NEXT:    lsl w14, w15, #86021; CHECK-GI-NEXT:    lsl w15, w16, #86022; CHECK-GI-NEXT:    ldr w16, [sp, #408]6023; CHECK-GI-NEXT:    lsl w10, w10, #86024; CHECK-GI-NEXT:    sbfx w12, w13, #8, #86025; CHECK-GI-NEXT:    ldr w13, [sp, #376]6026; CHECK-GI-NEXT:    mov v25.h[4], w96027; CHECK-GI-NEXT:    sbfx w9, w14, #8, #86028; CHECK-GI-NEXT:    sbfx w14, w15, #8, #86029; CHECK-GI-NEXT:    lsl w15, w16, #86030; CHECK-GI-NEXT:    mov v26.h[3], w126031; CHECK-GI-NEXT:    ldr w12, [sp, #416]6032; CHECK-GI-NEXT:    lsl w13, w13, #86033; CHECK-GI-NEXT:    sbfx w15, w15, #8, #86034; CHECK-GI-NEXT:    lsl w16, w17, #86035; CHECK-GI-NEXT:    mov v27.h[5], w96036; CHECK-GI-NEXT:    sbfx w13, w13, #8, #86037; CHECK-GI-NEXT:    lsl w12, w12, #86038; CHECK-GI-NEXT:    mov v25.h[5], w146039; CHECK-GI-NEXT:    fmov s29, w156040; CHECK-GI-NEXT:    ldr w14, [sp, #384]6041; CHECK-GI-NEXT:    ldr w15, [sp, #472]6042; CHECK-GI-NEXT:    mov v26.h[4], w136043; CHECK-GI-NEXT:    ldr w13, [sp, #424]6044; CHECK-GI-NEXT:    sbfx w12, w12, #8, #86045; CHECK-GI-NEXT:    sbfx w16, w16, #8, #86046; CHECK-GI-NEXT:    lsl w14, w14, #86047; CHECK-GI-NEXT:    lsl w15, w15, #86048; CHECK-GI-NEXT:    lsl w13, w13, #86049; CHECK-GI-NEXT:    mov v29.h[1], w126050; CHECK-GI-NEXT:    ldr w12, [sp, #480]6051; CHECK-GI-NEXT:    sbfx w14, w14, #8, #86052; CHECK-GI-NEXT:    mov v25.h[6], w166053; CHECK-GI-NEXT:    ldr w16, [sp, #432]6054; CHECK-GI-NEXT:    sbfx w13, w13, #8, #86055; CHECK-GI-NEXT:    sbfx w15, w15, #8, #86056; CHECK-GI-NEXT:    lsl w12, w12, #86057; CHECK-GI-NEXT:    mov v26.h[5], w146058; CHECK-GI-NEXT:    ldr w14, [sp, #392]6059; CHECK-GI-NEXT:    lsl w16, w16, #86060; CHECK-GI-NEXT:    mov v29.h[2], w136061; CHECK-GI-NEXT:    fmov s28, w156062; CHECK-GI-NEXT:    ldr w9, [sp, #336]6063; CHECK-GI-NEXT:    ldr w13, [sp, #488]6064; CHECK-GI-NEXT:    sbfx w12, w12, #8, #86065; CHECK-GI-NEXT:    lsl w14, w14, #86066; CHECK-GI-NEXT:    ldr w15, [sp, #440]6067; CHECK-GI-NEXT:    sbfx w16, w16, #8, #86068; CHECK-GI-NEXT:    lsl w9, w9, #86069; CHECK-GI-NEXT:    lsl w13, w13, #86070; CHECK-GI-NEXT:    mov v28.h[1], w126071; CHECK-GI-NEXT:    sbfx w14, w14, #8, #86072; CHECK-GI-NEXT:    lsl w15, w15, #86073; CHECK-GI-NEXT:    mov v29.h[3], w166074; CHECK-GI-NEXT:    ldr w16, [sp, #496]6075; CHECK-GI-NEXT:    sbfx w13, w13, #8, #86076; CHECK-GI-NEXT:    sbfx w9, w9, #8, #86077; CHECK-GI-NEXT:    ldr w12, [sp, #400]6078; CHECK-GI-NEXT:    mov v26.h[6], w146079; CHECK-GI-NEXT:    ldr w14, [sp, #448]6080; CHECK-GI-NEXT:    sbfx w15, w15, #8, #86081; CHECK-GI-NEXT:    mov v28.h[2], w136082; CHECK-GI-NEXT:    lsl w16, w16, #86083; CHECK-GI-NEXT:    mov v25.h[7], w96084; CHECK-GI-NEXT:    lsl w12, w12, #86085; CHECK-GI-NEXT:    mov v29.h[4], w156086; CHECK-GI-NEXT:    lsl w14, w14, #86087; CHECK-GI-NEXT:    ldr w13, [sp, #456]6088; CHECK-GI-NEXT:    ldr w15, [sp, #504]6089; CHECK-GI-NEXT:    sbfx w16, w16, #8, #86090; CHECK-GI-NEXT:    sbfx w9, w12, #8, #86091; CHECK-GI-NEXT:    sbfx w12, w14, #8, #86092; CHECK-GI-NEXT:    lsl w11, w11, #86093; CHECK-GI-NEXT:    lsl w14, w15, #86094; CHECK-GI-NEXT:    mov v28.h[3], w166095; CHECK-GI-NEXT:    ldr w15, [sp, #512]6096; CHECK-GI-NEXT:    lsl w13, w13, #86097; CHECK-GI-NEXT:    mul v30.8h, v22.8h, v25.8h6098; CHECK-GI-NEXT:    mov v26.h[7], w96099; CHECK-GI-NEXT:    mov v29.h[5], w126100; CHECK-GI-NEXT:    lsl w8, w8, #86101; CHECK-GI-NEXT:    sbfx w9, w14, #8, #86102; CHECK-GI-NEXT:    sbfx w10, w10, #8, #86103; CHECK-GI-NEXT:    sbfx w14, w11, #8, #86104; CHECK-GI-NEXT:    sbfx w11, w13, #8, #86105; CHECK-GI-NEXT:    lsl w13, w15, #86106; CHECK-GI-NEXT:    ldr w17, [sp, #464]6107; CHECK-GI-NEXT:    sbfx w8, w8, #8, #86108; CHECK-GI-NEXT:    mov v28.h[4], w96109; CHECK-GI-NEXT:    mov v27.h[6], w106110; CHECK-GI-NEXT:    ldr w16, [sp, #520]6111; CHECK-GI-NEXT:    sbfx w10, w13, #8, #86112; CHECK-GI-NEXT:    smov w13, v30.h[0]6113; CHECK-GI-NEXT:    mov v24.h[7], w86114; CHECK-GI-NEXT:    lsl w8, w17, #86115; CHECK-GI-NEXT:    mov v29.h[6], w116116; CHECK-GI-NEXT:    mul v26.8h, v23.8h, v26.8h6117; CHECK-GI-NEXT:    lsl w15, w16, #86118; CHECK-GI-NEXT:    smov w16, v30.h[1]6119; CHECK-GI-NEXT:    ldr w12, [sp, #528]6120; CHECK-GI-NEXT:    sbfx w8, w8, #8, #86121; CHECK-GI-NEXT:    mov v28.h[5], w106122; CHECK-GI-NEXT:    mov v27.h[7], w146123; CHECK-GI-NEXT:    fmov s22, w136124; CHECK-GI-NEXT:    sbfx w10, w15, #8, #86125; CHECK-GI-NEXT:    smov w14, v30.h[4]6126; CHECK-GI-NEXT:    mov v29.h[7], w86127; CHECK-GI-NEXT:    smov w15, v26.h[0]6128; CHECK-GI-NEXT:    smov w13, v30.h[2]6129; CHECK-GI-NEXT:    lsl w12, w12, #86130; CHECK-GI-NEXT:    ldr w9, [sp, #544]6131; CHECK-GI-NEXT:    ldr w11, [sp, #552]6132; CHECK-GI-NEXT:    mov v22.s[1], w166133; CHECK-GI-NEXT:    smov w16, v26.h[4]6134; CHECK-GI-NEXT:    mov v28.h[6], w106135; CHECK-GI-NEXT:    smov w10, v26.h[1]6136; CHECK-GI-NEXT:    fmov s23, w146137; CHECK-GI-NEXT:    smov w14, v26.h[5]6138; CHECK-GI-NEXT:    mul v29.8h, v24.8h, v29.8h6139; CHECK-GI-NEXT:    fmov s24, w156140; CHECK-GI-NEXT:    smov w15, v26.h[2]6141; CHECK-GI-NEXT:    lsl w11, w11, #86142; CHECK-GI-NEXT:    smov w8, v30.h[5]6143; CHECK-GI-NEXT:    smov w17, v30.h[7]6144; CHECK-GI-NEXT:    fmov s25, w166145; CHECK-GI-NEXT:    mov v22.s[2], w136146; CHECK-GI-NEXT:    smov w13, v30.h[3]6147; CHECK-GI-NEXT:    mov v24.s[1], w106148; CHECK-GI-NEXT:    smov w16, v26.h[6]6149; CHECK-GI-NEXT:    sbfx w10, w12, #8, #86150; CHECK-GI-NEXT:    smov w18, v29.h[0]6151; CHECK-GI-NEXT:    smov w0, v29.h[1]6152; CHECK-GI-NEXT:    ldr w12, [sp, #560]6153; CHECK-GI-NEXT:    mov v25.s[1], w146154; CHECK-GI-NEXT:    smov w14, v26.h[7]6155; CHECK-GI-NEXT:    mov v28.h[7], w106156; CHECK-GI-NEXT:    mov v22.s[3], w136157; CHECK-GI-NEXT:    smov w13, v26.h[3]6158; CHECK-GI-NEXT:    sbfx w11, w11, #8, #86159; CHECK-GI-NEXT:    mov v24.s[2], w156160; CHECK-GI-NEXT:    smov w15, v29.h[2]6161; CHECK-GI-NEXT:    lsl w12, w12, #86162; CHECK-GI-NEXT:    fmov s26, w186163; CHECK-GI-NEXT:    mov v23.s[1], w86164; CHECK-GI-NEXT:    smov w8, v30.h[6]6165; CHECK-GI-NEXT:    mov v25.s[2], w166166; CHECK-GI-NEXT:    lsl w16, w9, #86167; CHECK-GI-NEXT:    mul v31.8h, v27.8h, v28.8h6168; CHECK-GI-NEXT:    ldr w10, [sp, #568]6169; CHECK-GI-NEXT:    sbfx w12, w12, #8, #86170; CHECK-GI-NEXT:    ldr w9, [sp, #584]6171; CHECK-GI-NEXT:    mov v24.s[3], w136172; CHECK-GI-NEXT:    smov w13, v29.h[4]6173; CHECK-GI-NEXT:    mov v26.s[1], w06174; CHECK-GI-NEXT:    sbfx w16, w16, #8, #86175; CHECK-GI-NEXT:    lsl w10, w10, #86176; CHECK-GI-NEXT:    mov v23.s[2], w86177; CHECK-GI-NEXT:    mov v25.s[3], w146178; CHECK-GI-NEXT:    ldr w14, [sp, #608]6179; CHECK-GI-NEXT:    ldr w8, [sp, #576]6180; CHECK-GI-NEXT:    fmov s8, w166181; CHECK-GI-NEXT:    ldr w16, [sp, #616]6182; CHECK-GI-NEXT:    sbfx w10, w10, #8, #86183; CHECK-GI-NEXT:    fmov s27, w136184; CHECK-GI-NEXT:    lsl w13, w14, #86185; CHECK-GI-NEXT:    mov v26.s[2], w156186; CHECK-GI-NEXT:    smov w15, v29.h[5]6187; CHECK-GI-NEXT:    lsl w16, w16, #86188; CHECK-GI-NEXT:    ldr w14, [sp, #624]6189; CHECK-GI-NEXT:    sbfx w13, w13, #8, #86190; CHECK-GI-NEXT:    mov v8.h[1], w116191; CHECK-GI-NEXT:    lsl w8, w8, #86192; CHECK-GI-NEXT:    sbfx w16, w16, #8, #86193; CHECK-GI-NEXT:    lsl w14, w14, #86194; CHECK-GI-NEXT:    mov v23.s[3], w176195; CHECK-GI-NEXT:    fmov s9, w136196; CHECK-GI-NEXT:    ldr w13, [sp, #632]6197; CHECK-GI-NEXT:    smov w17, v31.h[1]6198; CHECK-GI-NEXT:    mov v27.s[1], w156199; CHECK-GI-NEXT:    smov w15, v31.h[0]6200; CHECK-GI-NEXT:    sbfx w14, w14, #8, #86201; CHECK-GI-NEXT:    mov v8.h[2], w126202; CHECK-GI-NEXT:    lsl w13, w13, #86203; CHECK-GI-NEXT:    sbfx w8, w8, #8, #86204; CHECK-GI-NEXT:    mov v9.h[1], w166205; CHECK-GI-NEXT:    smov w16, v31.h[2]6206; CHECK-GI-NEXT:    lsl w9, w9, #86207; CHECK-GI-NEXT:    sbfx w13, w13, #8, #86208; CHECK-GI-NEXT:    ldr w11, [sp, #592]6209; CHECK-GI-NEXT:    ldr w12, [sp, #600]6210; CHECK-GI-NEXT:    fmov s28, w156211; CHECK-GI-NEXT:    smov w15, v29.h[6]6212; CHECK-GI-NEXT:    sbfx w9, w9, #8, #86213; CHECK-GI-NEXT:    mov v8.h[3], w106214; CHECK-GI-NEXT:    ldr w10, [sp, #640]6215; CHECK-GI-NEXT:    lsl w11, w11, #86216; CHECK-GI-NEXT:    mov v9.h[2], w146217; CHECK-GI-NEXT:    ldr w14, [sp, #672]6218; CHECK-GI-NEXT:    lsl w12, w12, #86219; CHECK-GI-NEXT:    mov v28.s[1], w176220; CHECK-GI-NEXT:    lsl w10, w10, #86221; CHECK-GI-NEXT:    sbfx w11, w11, #8, #86222; CHECK-GI-NEXT:    lsl w14, w14, #86223; CHECK-GI-NEXT:    mov v27.s[2], w156224; CHECK-GI-NEXT:    ldr w15, [sp, #680]6225; CHECK-GI-NEXT:    mov v8.h[4], w86226; CHECK-GI-NEXT:    smov w8, v31.h[4]6227; CHECK-GI-NEXT:    sbfx w12, w12, #8, #86228; CHECK-GI-NEXT:    sbfx w14, w14, #8, #86229; CHECK-GI-NEXT:    lsl w15, w15, #86230; CHECK-GI-NEXT:    mov v9.h[3], w136231; CHECK-GI-NEXT:    ldr w13, [sp, #688]6232; CHECK-GI-NEXT:    mov v28.s[2], w166233; CHECK-GI-NEXT:    ldr w16, [sp, #648]6234; CHECK-GI-NEXT:    fmov s10, w146235; CHECK-GI-NEXT:    sbfx w15, w15, #8, #86236; CHECK-GI-NEXT:    ldr w14, [sp, #656]6237; CHECK-GI-NEXT:    lsl w13, w13, #86238; CHECK-GI-NEXT:    fmov s30, w86239; CHECK-GI-NEXT:    sbfx w8, w10, #8, #86240; CHECK-GI-NEXT:    smov w10, v31.h[5]6241; CHECK-GI-NEXT:    mov v8.h[5], w96242; CHECK-GI-NEXT:    ldr w9, [sp, #696]6243; CHECK-GI-NEXT:    mov v10.h[1], w156244; CHECK-GI-NEXT:    sbfx w13, w13, #8, #86245; CHECK-GI-NEXT:    mov v9.h[4], w86246; CHECK-GI-NEXT:    lsl w16, w16, #86247; CHECK-GI-NEXT:    lsl w9, w9, #86248; CHECK-GI-NEXT:    lsl w14, w14, #86249; CHECK-GI-NEXT:    ldr w8, [sp, #704]6250; CHECK-GI-NEXT:    ldr w15, [sp, #664]6251; CHECK-GI-NEXT:    ldr w17, [sp, #768]6252; CHECK-GI-NEXT:    mov v30.s[1], w106253; CHECK-GI-NEXT:    ldr w10, [sp, #744]6254; CHECK-GI-NEXT:    sbfx w16, w16, #8, #86255; CHECK-GI-NEXT:    mov v10.h[2], w136256; CHECK-GI-NEXT:    ldr w13, [sp, #736]6257; CHECK-GI-NEXT:    sbfx w9, w9, #8, #86258; CHECK-GI-NEXT:    lsl w10, w10, #86259; CHECK-GI-NEXT:    mov v9.h[5], w166260; CHECK-GI-NEXT:    mov v8.h[6], w116261; CHECK-GI-NEXT:    lsl w13, w13, #86262; CHECK-GI-NEXT:    ldr w11, [sp, #712]6263; CHECK-GI-NEXT:    lsl w8, w8, #86264; CHECK-GI-NEXT:    sbfx w10, w10, #8, #86265; CHECK-GI-NEXT:    ldr w16, [sp, #720]6266; CHECK-GI-NEXT:    lsl w15, w15, #86267; CHECK-GI-NEXT:    sbfx w13, w13, #8, #86268; CHECK-GI-NEXT:    mov v10.h[3], w96269; CHECK-GI-NEXT:    ldr w9, [sp, #752]6270; CHECK-GI-NEXT:    mov v8.h[7], w126271; CHECK-GI-NEXT:    sbfx w12, w8, #8, #86272; CHECK-GI-NEXT:    lsl w18, w16, #86273; CHECK-GI-NEXT:    fmov s11, w136274; CHECK-GI-NEXT:    ldr w13, [sp, #760]6275; CHECK-GI-NEXT:    ldr w8, [sp, #784]6276; CHECK-GI-NEXT:    mov v21.s[1], wzr6277; CHECK-GI-NEXT:    mov v16.s[1], wzr6278; CHECK-GI-NEXT:    mov v18.s[1], wzr6279; CHECK-GI-NEXT:    lsl w13, w13, #86280; CHECK-GI-NEXT:    mov v10.h[4], w126281; CHECK-GI-NEXT:    sbfx w12, w15, #8, #86282; CHECK-GI-NEXT:    mov v11.h[1], w106283; CHECK-GI-NEXT:    sbfx w10, w14, #8, #86284; CHECK-GI-NEXT:    lsl w14, w9, #86285; CHECK-GI-NEXT:    sbfx w13, w13, #8, #86286; CHECK-GI-NEXT:    ldr w9, [sp, #776]6287; CHECK-GI-NEXT:    lsl w8, w8, #86288; CHECK-GI-NEXT:    sbfx w14, w14, #8, #86289; CHECK-GI-NEXT:    mov v9.h[6], w106290; CHECK-GI-NEXT:    lsl w10, w11, #86291; CHECK-GI-NEXT:    ldr w11, [sp, #808]6292; CHECK-GI-NEXT:    lsl w9, w9, #86293; CHECK-GI-NEXT:    sbfx w8, w8, #8, #86294; CHECK-GI-NEXT:    mov v11.h[2], w146295; CHECK-GI-NEXT:    ldr w14, [sp, #816]6296; CHECK-GI-NEXT:    sbfx w10, w10, #8, #86297; CHECK-GI-NEXT:    lsl w11, w11, #86298; CHECK-GI-NEXT:    sbfx w9, w9, #8, #86299; CHECK-GI-NEXT:    mov v17.s[1], wzr6300; CHECK-GI-NEXT:    lsl w14, w14, #86301; CHECK-GI-NEXT:    mov v9.h[7], w126302; CHECK-GI-NEXT:    ldr w12, [sp, #824]6303; CHECK-GI-NEXT:    sbfx w16, w11, #8, #86304; CHECK-GI-NEXT:    mov v10.h[5], w106305; CHECK-GI-NEXT:    ldr w10, [sp, #832]6306; CHECK-GI-NEXT:    mov v11.h[3], w136307; CHECK-GI-NEXT:    sbfx w15, w14, #8, #86308; CHECK-GI-NEXT:    lsl w14, w17, #86309; CHECK-GI-NEXT:    fmov s12, w166310; CHECK-GI-NEXT:    ldr w16, [sp, #872]6311; CHECK-GI-NEXT:    lsl w12, w12, #86312; CHECK-GI-NEXT:    sbfx w14, w14, #8, #86313; CHECK-GI-NEXT:    ldr w11, [sp, #840]6314; CHECK-GI-NEXT:    sbfx w13, w18, #8, #86315; CHECK-GI-NEXT:    sbfx w17, w12, #8, #86316; CHECK-GI-NEXT:    lsl w16, w16, #86317; CHECK-GI-NEXT:    ldr w12, [sp, #856]6318; CHECK-GI-NEXT:    mov v12.h[1], w156319; CHECK-GI-NEXT:    mov v11.h[4], w146320; CHECK-GI-NEXT:    ldr w15, [sp, #880]6321; CHECK-GI-NEXT:    lsl w11, w11, #86322; CHECK-GI-NEXT:    mov v10.h[6], w136323; CHECK-GI-NEXT:    ldr w13, [sp, #848]6324; CHECK-GI-NEXT:    lsl w14, w15, #86325; CHECK-GI-NEXT:    sbfx w15, w16, #8, #86326; CHECK-GI-NEXT:    ldr w16, [sp, #888]6327; CHECK-GI-NEXT:    lsl w13, w13, #86328; CHECK-GI-NEXT:    lsl w12, w12, #86329; CHECK-GI-NEXT:    mov v20.s[1], wzr6330; CHECK-GI-NEXT:    mov v12.h[2], w176331; CHECK-GI-NEXT:    lsl w17, w10, #86332; CHECK-GI-NEXT:    mov v11.h[5], w96333; CHECK-GI-NEXT:    fmov s13, w156334; CHECK-GI-NEXT:    ldr w9, [sp, #936]6335; CHECK-GI-NEXT:    sbfx w14, w14, #8, #86336; CHECK-GI-NEXT:    sbfx w15, w17, #8, #86337; CHECK-GI-NEXT:    lsl w16, w16, #86338; CHECK-GI-NEXT:    sbfx w13, w13, #8, #86339; CHECK-GI-NEXT:    lsl w9, w9, #86340; CHECK-GI-NEXT:    sbfx w12, w12, #8, #86341; CHECK-GI-NEXT:    ldr w10, [sp, #864]6342; CHECK-GI-NEXT:    mov v12.h[3], w156343; CHECK-GI-NEXT:    mov v11.h[6], w86344; CHECK-GI-NEXT:    sbfx w8, w11, #8, #86345; CHECK-GI-NEXT:    ldr w11, [sp, #1000]6346; CHECK-GI-NEXT:    mov v13.h[1], w146347; CHECK-GI-NEXT:    ldr w15, [sp, #944]6348; CHECK-GI-NEXT:    sbfx w9, w9, #8, #86349; CHECK-GI-NEXT:    ldr w14, [sp, #896]6350; CHECK-GI-NEXT:    sbfx w16, w16, #8, #86351; CHECK-GI-NEXT:    lsl w11, w11, #86352; CHECK-GI-NEXT:    lsl w15, w15, #86353; CHECK-GI-NEXT:    lsl w10, w10, #86354; CHECK-GI-NEXT:    mov v12.h[4], w86355; CHECK-GI-NEXT:    ldr w8, [sp, #1008]6356; CHECK-GI-NEXT:    fmov s14, w96357; CHECK-GI-NEXT:    sbfx w11, w11, #8, #86358; CHECK-GI-NEXT:    mov v13.h[2], w166359; CHECK-GI-NEXT:    ldr w16, [sp, #952]6360; CHECK-GI-NEXT:    lsl w14, w14, #86361; CHECK-GI-NEXT:    sbfx w15, w15, #8, #86362; CHECK-GI-NEXT:    lsl w17, w8, #86363; CHECK-GI-NEXT:    smov w8, v29.h[3]6364; CHECK-GI-NEXT:    smov w9, v29.h[7]6365; CHECK-GI-NEXT:    fmov s29, w116366; CHECK-GI-NEXT:    sbfx w14, w14, #8, #86367; CHECK-GI-NEXT:    mov v14.h[1], w156368; CHECK-GI-NEXT:    sbfx w15, w17, #8, #86369; CHECK-GI-NEXT:    ldr w11, [sp, #904]6370; CHECK-GI-NEXT:    lsl w16, w16, #86371; CHECK-GI-NEXT:    mov v12.h[5], w136372; CHECK-GI-NEXT:    mov v13.h[3], w146373; CHECK-GI-NEXT:    mov v29.h[1], w156374; CHECK-GI-NEXT:    ldr w15, [sp, #960]6375; CHECK-GI-NEXT:    lsl w11, w11, #86376; CHECK-GI-NEXT:    sbfx w16, w16, #8, #86377; CHECK-GI-NEXT:    ldr w14, [sp, #1016]6378; CHECK-GI-NEXT:    lsl w15, w15, #86379; CHECK-GI-NEXT:    ldr w13, [sp, #1024]6380; CHECK-GI-NEXT:    sbfx w10, w10, #8, #86381; CHECK-GI-NEXT:    sbfx w11, w11, #8, #86382; CHECK-GI-NEXT:    mov v14.h[2], w166383; CHECK-GI-NEXT:    lsl w14, w14, #86384; CHECK-GI-NEXT:    sbfx w15, w15, #8, #86385; CHECK-GI-NEXT:    ldr w16, [sp, #912]6386; CHECK-GI-NEXT:    lsl w13, w13, #86387; CHECK-GI-NEXT:    mov v13.h[4], w116388; CHECK-GI-NEXT:    ldr w11, [sp, #968]6389; CHECK-GI-NEXT:    sbfx w14, w14, #8, #86390; CHECK-GI-NEXT:    mov v12.h[6], w126391; CHECK-GI-NEXT:    ldr w12, [sp, #976]6392; CHECK-GI-NEXT:    sbfx w13, w13, #8, #86393; CHECK-GI-NEXT:    mov v14.h[3], w156394; CHECK-GI-NEXT:    lsl w11, w11, #86395; CHECK-GI-NEXT:    mov v29.h[2], w146396; CHECK-GI-NEXT:    ldr w15, [sp, #1032]6397; CHECK-GI-NEXT:    lsl w16, w16, #86398; CHECK-GI-NEXT:    lsl w12, w12, #86399; CHECK-GI-NEXT:    sbfx w11, w11, #8, #86400; CHECK-GI-NEXT:    ldr w14, [sp, #920]6401; CHECK-GI-NEXT:    mov v26.s[3], w86402; CHECK-GI-NEXT:    sbfx w16, w16, #8, #86403; CHECK-GI-NEXT:    lsl w15, w15, #86404; CHECK-GI-NEXT:    sbfx w12, w12, #8, #86405; CHECK-GI-NEXT:    mov v14.h[4], w116406; CHECK-GI-NEXT:    mov v29.h[3], w136407; CHECK-GI-NEXT:    ldr w11, [sp, #984]6408; CHECK-GI-NEXT:    lsl w14, w14, #86409; CHECK-GI-NEXT:    sbfx w15, w15, #8, #86410; CHECK-GI-NEXT:    mov v13.h[5], w166411; CHECK-GI-NEXT:    ldr w16, [sp, #1040]6412; CHECK-GI-NEXT:    lsl w11, w11, #86413; CHECK-GI-NEXT:    ldr w13, [sp, #928]6414; CHECK-GI-NEXT:    sbfx w14, w14, #8, #86415; CHECK-GI-NEXT:    mov v12.h[7], w106416; CHECK-GI-NEXT:    mov v27.s[3], w96417; CHECK-GI-NEXT:    mov v14.h[5], w126418; CHECK-GI-NEXT:    mov v29.h[4], w156419; CHECK-GI-NEXT:    lsl w16, w16, #86420; CHECK-GI-NEXT:    sbfx w10, w11, #8, #86421; CHECK-GI-NEXT:    lsl w13, w13, #86422; CHECK-GI-NEXT:    mov v13.h[6], w146423; CHECK-GI-NEXT:    ldr w12, [sp, #1048]6424; CHECK-GI-NEXT:    sbfx w14, w16, #8, #86425; CHECK-GI-NEXT:    ldr w11, [sp, #728]6426; CHECK-GI-NEXT:    sbfx w13, w13, #8, #86427; CHECK-GI-NEXT:    mul v15.8h, v8.8h, v12.8h6428; CHECK-GI-NEXT:    smov w16, v31.h[6]6429; CHECK-GI-NEXT:    mov v14.h[6], w106430; CHECK-GI-NEXT:    ldr w10, [sp, #992]6431; CHECK-GI-NEXT:    mov v29.h[5], w146432; CHECK-GI-NEXT:    lsl w12, w12, #86433; CHECK-GI-NEXT:    lsl w11, w11, #86434; CHECK-GI-NEXT:    mov v13.h[7], w136435; CHECK-GI-NEXT:    lsl w10, w10, #86436; CHECK-GI-NEXT:    ldr w13, [sp, #792]6437; CHECK-GI-NEXT:    ldr w14, [sp, #1056]6438; CHECK-GI-NEXT:    sbfx w12, w12, #8, #86439; CHECK-GI-NEXT:    sbfx w11, w11, #8, #86440; CHECK-GI-NEXT:    mov v30.s[2], w166441; CHECK-GI-NEXT:    sbfx w10, w10, #8, #86442; CHECK-GI-NEXT:    smov w8, v15.h[1]6443; CHECK-GI-NEXT:    smov w9, v15.h[5]6444; CHECK-GI-NEXT:    mov v29.h[6], w126445; CHECK-GI-NEXT:    lsl w12, w13, #86446; CHECK-GI-NEXT:    lsl w13, w14, #86447; CHECK-GI-NEXT:    mov v10.h[7], w116448; CHECK-GI-NEXT:    mov v14.h[7], w106449; CHECK-GI-NEXT:    mul v12.8h, v9.8h, v13.8h6450; CHECK-GI-NEXT:    sbfx w12, w12, #8, #86451; CHECK-GI-NEXT:    sbfx w13, w13, #8, #86452; CHECK-GI-NEXT:    smov w10, v15.h[0]6453; CHECK-GI-NEXT:    smov w11, v15.h[4]6454; CHECK-GI-NEXT:    smov w14, v31.h[7]6455; CHECK-GI-NEXT:    smov w15, v31.h[3]6456; CHECK-GI-NEXT:    mov v11.h[7], w126457; CHECK-GI-NEXT:    mov v29.h[7], w136458; CHECK-GI-NEXT:    mov v6.s[1], wzr6459; CHECK-GI-NEXT:    mul v13.8h, v10.8h, v14.8h6460; CHECK-GI-NEXT:    smov w12, v12.h[0]6461; CHECK-GI-NEXT:    smov w13, v12.h[1]6462; CHECK-GI-NEXT:    mov v7.s[1], wzr6463; CHECK-GI-NEXT:    mov v2.s[1], wzr6464; CHECK-GI-NEXT:    mov v4.s[1], wzr6465; CHECK-GI-NEXT:    fmov s31, w116466; CHECK-GI-NEXT:    mov v30.s[3], w146467; CHECK-GI-NEXT:    smov w11, v12.h[4]6468; CHECK-GI-NEXT:    mul v14.8h, v11.8h, v29.8h6469; CHECK-GI-NEXT:    fmov s29, w106470; CHECK-GI-NEXT:    smov w10, v15.h[2]6471; CHECK-GI-NEXT:    smov w14, v13.h[0]6472; CHECK-GI-NEXT:    fmov s8, w126473; CHECK-GI-NEXT:    smov w16, v13.h[1]6474; CHECK-GI-NEXT:    mov v31.s[1], w96475; CHECK-GI-NEXT:    smov w9, v12.h[2]6476; CHECK-GI-NEXT:    mov v28.s[3], w156477; CHECK-GI-NEXT:    mov v29.s[1], w86478; CHECK-GI-NEXT:    smov w8, v15.h[6]6479; CHECK-GI-NEXT:    smov w15, v12.h[5]6480; CHECK-GI-NEXT:    mov v8.s[1], w136481; CHECK-GI-NEXT:    fmov s9, w116482; CHECK-GI-NEXT:    smov w12, v15.h[3]6483; CHECK-GI-NEXT:    fmov s10, w146484; CHECK-GI-NEXT:    smov w14, v13.h[2]6485; CHECK-GI-NEXT:    smov w11, v12.h[6]6486; CHECK-GI-NEXT:    smov w13, v15.h[7]6487; CHECK-GI-NEXT:    mov v3.s[1], wzr6488; CHECK-GI-NEXT:    mov v5.s[1], wzr6489; CHECK-GI-NEXT:    mov v31.s[2], w86490; CHECK-GI-NEXT:    smov w8, v13.h[4]6491; CHECK-GI-NEXT:    mov v29.s[2], w106492; CHECK-GI-NEXT:    mov v10.s[1], w166493; CHECK-GI-NEXT:    smov w16, v14.h[0]6494; CHECK-GI-NEXT:    mov v8.s[2], w96495; CHECK-GI-NEXT:    smov w9, v13.h[5]6496; CHECK-GI-NEXT:    smov w10, v12.h[3]6497; CHECK-GI-NEXT:    mov v9.s[1], w156498; CHECK-GI-NEXT:    smov w15, v13.h[6]6499; CHECK-GI-NEXT:    mov v1.s[1], wzr6500; CHECK-GI-NEXT:    mov v0.s[1], wzr6501; CHECK-GI-NEXT:    fmov s11, w86502; CHECK-GI-NEXT:    smov w8, v14.h[1]6503; CHECK-GI-NEXT:    mov v29.s[3], w126504; CHECK-GI-NEXT:    mov v10.s[2], w146505; CHECK-GI-NEXT:    smov w14, v12.h[7]6506; CHECK-GI-NEXT:    fmov s12, w166507; CHECK-GI-NEXT:    smov w12, v14.h[4]6508; CHECK-GI-NEXT:    mov v8.s[3], w106509; CHECK-GI-NEXT:    ldr w10, [sp, #536]6510; CHECK-GI-NEXT:    mov v11.s[1], w96511; CHECK-GI-NEXT:    ldr w9, [sp, #272]6512; CHECK-GI-NEXT:    mov v9.s[2], w116513; CHECK-GI-NEXT:    ldr w11, [sp, #800]6514; CHECK-GI-NEXT:    mov v12.s[1], w86515; CHECK-GI-NEXT:    ldr w8, [sp, #1064]6516; CHECK-GI-NEXT:    mov v31.s[3], w136517; CHECK-GI-NEXT:    smov w13, v14.h[5]6518; CHECK-GI-NEXT:    sxtb w9, w96519; CHECK-GI-NEXT:    sxtb w10, w106520; CHECK-GI-NEXT:    sxtb w11, w116521; CHECK-GI-NEXT:    sxtb w8, w86522; CHECK-GI-NEXT:    mov v11.s[2], w156523; CHECK-GI-NEXT:    smov w15, v13.h[3]6524; CHECK-GI-NEXT:    smov w16, v13.h[7]6525; CHECK-GI-NEXT:    fmov s13, w126526; CHECK-GI-NEXT:    mul w9, w9, w106527; CHECK-GI-NEXT:    smov w12, v14.h[2]6528; CHECK-GI-NEXT:    mul w8, w11, w86529; CHECK-GI-NEXT:    mov v19.s[2], wzr6530; CHECK-GI-NEXT:    mov v21.s[2], wzr6531; CHECK-GI-NEXT:    mov v16.s[2], wzr6532; CHECK-GI-NEXT:    mov v18.s[2], wzr6533; CHECK-GI-NEXT:    mov v17.s[2], wzr6534; CHECK-GI-NEXT:    mov v13.s[1], w136535; CHECK-GI-NEXT:    smov w13, v14.h[6]6536; CHECK-GI-NEXT:    sxth w9, w96537; CHECK-GI-NEXT:    sxth w10, w86538; CHECK-GI-NEXT:    mov v20.s[2], wzr6539; CHECK-GI-NEXT:    mov v6.s[2], wzr6540; CHECK-GI-NEXT:    mov v7.s[2], wzr6541; CHECK-GI-NEXT:    mov v2.s[2], wzr6542; CHECK-GI-NEXT:    mov v4.s[2], wzr6543; CHECK-GI-NEXT:    mov v3.s[2], wzr6544; CHECK-GI-NEXT:    mov v5.s[2], wzr6545; CHECK-GI-NEXT:    add v22.4s, v22.4s, v23.4s6546; CHECK-GI-NEXT:    add v25.4s, v24.4s, v25.4s6547; CHECK-GI-NEXT:    fmov s23, w96548; CHECK-GI-NEXT:    fmov s24, w106549; CHECK-GI-NEXT:    mov v12.s[2], w126550; CHECK-GI-NEXT:    mov v13.s[2], w136551; CHECK-GI-NEXT:    smov w8, v14.h[3]6552; CHECK-GI-NEXT:    smov w9, v14.h[7]6553; CHECK-GI-NEXT:    mov v1.s[2], wzr6554; CHECK-GI-NEXT:    mov v0.s[2], wzr6555; CHECK-GI-NEXT:    mov v19.s[3], wzr6556; CHECK-GI-NEXT:    mov v21.s[3], wzr6557; CHECK-GI-NEXT:    mov v16.s[3], wzr6558; CHECK-GI-NEXT:    mov v18.s[3], wzr6559; CHECK-GI-NEXT:    mov v17.s[3], wzr6560; CHECK-GI-NEXT:    mov v20.s[3], wzr6561; CHECK-GI-NEXT:    mov v6.s[3], wzr6562; CHECK-GI-NEXT:    mov v7.s[3], wzr6563; CHECK-GI-NEXT:    mov v2.s[3], wzr6564; CHECK-GI-NEXT:    mov v4.s[3], wzr6565; CHECK-GI-NEXT:    mov v3.s[3], wzr6566; CHECK-GI-NEXT:    mov v5.s[3], wzr6567; CHECK-GI-NEXT:    mov v23.s[1], wzr6568; CHECK-GI-NEXT:    mov v24.s[1], wzr6569; CHECK-GI-NEXT:    mov v9.s[3], w146570; CHECK-GI-NEXT:    mov v10.s[3], w156571; CHECK-GI-NEXT:    mov v11.s[3], w166572; CHECK-GI-NEXT:    mov v1.s[3], wzr6573; CHECK-GI-NEXT:    mov v12.s[3], w86574; CHECK-GI-NEXT:    mov v13.s[3], w96575; CHECK-GI-NEXT:    mov v0.s[3], wzr6576; CHECK-GI-NEXT:    add v19.4s, v19.4s, v21.4s6577; CHECK-GI-NEXT:    add v16.4s, v16.4s, v18.4s6578; CHECK-GI-NEXT:    add v17.4s, v17.4s, v20.4s6579; CHECK-GI-NEXT:    add v6.4s, v6.4s, v7.4s6580; CHECK-GI-NEXT:    add v2.4s, v2.4s, v4.4s6581; CHECK-GI-NEXT:    add v3.4s, v3.4s, v5.4s6582; CHECK-GI-NEXT:    mov v23.s[2], wzr6583; CHECK-GI-NEXT:    mov v24.s[2], wzr6584; CHECK-GI-NEXT:    add v26.4s, v26.4s, v27.4s6585; CHECK-GI-NEXT:    add v27.4s, v28.4s, v30.4s6586; CHECK-GI-NEXT:    add v1.4s, v1.4s, v19.4s6587; CHECK-GI-NEXT:    add v4.4s, v16.4s, v17.4s6588; CHECK-GI-NEXT:    add v5.4s, v29.4s, v31.4s6589; CHECK-GI-NEXT:    add v7.4s, v8.4s, v9.4s6590; CHECK-GI-NEXT:    add v16.4s, v10.4s, v11.4s6591; CHECK-GI-NEXT:    add v17.4s, v12.4s, v13.4s6592; CHECK-GI-NEXT:    add v0.4s, v0.4s, v6.4s6593; CHECK-GI-NEXT:    add v2.4s, v2.4s, v3.4s6594; CHECK-GI-NEXT:    mov v23.s[3], wzr6595; CHECK-GI-NEXT:    mov v24.s[3], wzr6596; CHECK-GI-NEXT:    add v3.4s, v22.4s, v25.4s6597; CHECK-GI-NEXT:    add v6.4s, v26.4s, v27.4s6598; CHECK-GI-NEXT:    add v1.4s, v1.4s, v4.4s6599; CHECK-GI-NEXT:    add v4.4s, v5.4s, v7.4s6600; CHECK-GI-NEXT:    add v5.4s, v16.4s, v17.4s6601; CHECK-GI-NEXT:    add v0.4s, v0.4s, v2.4s6602; CHECK-GI-NEXT:    ldr x29, [sp, #64] // 8-byte Reload6603; CHECK-GI-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload6604; CHECK-GI-NEXT:    add v2.4s, v3.4s, v6.4s6605; CHECK-GI-NEXT:    add v1.4s, v23.4s, v1.4s6606; CHECK-GI-NEXT:    add v3.4s, v4.4s, v5.4s6607; CHECK-GI-NEXT:    add v0.4s, v24.4s, v0.4s6608; CHECK-GI-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload6609; CHECK-GI-NEXT:    add v1.4s, v2.4s, v1.4s6610; CHECK-GI-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload6611; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s6612; CHECK-GI-NEXT:    addv s1, v1.4s6613; CHECK-GI-NEXT:    addv s0, v0.4s6614; CHECK-GI-NEXT:    fmov w8, s16615; CHECK-GI-NEXT:    fmov w9, s06616; CHECK-GI-NEXT:    add w0, w8, w96617; CHECK-GI-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload6618; CHECK-GI-NEXT:    ret6619entry:6620  %az = sext <33 x i8> %a to <33 x i32>6621  %bz = sext <33 x i8> %b to <33 x i32>6622  %m1 = mul nuw nsw <33 x i32> %az, %bz6623  %r1 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %m1)6624  %cz = sext <33 x i8> %c to <33 x i32>6625  %dz = sext <33 x i8> %d to <33 x i32>6626  %m2 = mul nuw nsw <33 x i32> %cz, %dz6627  %r2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %m2)6628  %x = add i32 %r1, %r26629  ret i32 %x6630}6631 6632define i32 @test_sdot_v33i8_double_nomla(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33 x i8> %d) {6633; CHECK-SD-LABEL: test_sdot_v33i8_double_nomla:6634; CHECK-SD:       // %bb.0: // %entry6635; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill6636; CHECK-SD-NEXT:    .cfi_def_cfa_offset 166637; CHECK-SD-NEXT:    .cfi_offset w29, -166638; CHECK-SD-NEXT:    ldr b0, [sp, #80]6639; CHECK-SD-NEXT:    add x8, sp, #886640; CHECK-SD-NEXT:    ldr b2, [sp, #144]6641; CHECK-SD-NEXT:    add x9, sp, #1526642; CHECK-SD-NEXT:    ldr b3, [sp, #16]6643; CHECK-SD-NEXT:    add x11, sp, #1046644; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]6645; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x9]6646; CHECK-SD-NEXT:    add x9, sp, #246647; CHECK-SD-NEXT:    add x8, sp, #966648; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x9]6649; CHECK-SD-NEXT:    ldr b5, [sp, #480]6650; CHECK-SD-NEXT:    fmov s1, w06651; CHECK-SD-NEXT:    add x10, sp, #1126652; CHECK-SD-NEXT:    add x12, sp, #1686653; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]6654; CHECK-SD-NEXT:    add x8, sp, #1606655; CHECK-SD-NEXT:    ldr b4, [sp, #608]6656; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x8]6657; CHECK-SD-NEXT:    add x8, sp, #326658; CHECK-SD-NEXT:    add x13, sp, #4966659; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x8]6660; CHECK-SD-NEXT:    mov v1.b[1], w16661; CHECK-SD-NEXT:    ldr b6, [sp, #672]6662; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x11]6663; CHECK-SD-NEXT:    add x11, sp, #4886664; CHECK-SD-NEXT:    add x9, sp, #1206665; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x11]6666; CHECK-SD-NEXT:    add x11, sp, #406667; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x12]6668; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x11]6669; CHECK-SD-NEXT:    add x12, sp, #6166670; CHECK-SD-NEXT:    ldr b16, [sp, #544]6671; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x10]6672; CHECK-SD-NEXT:    add x10, sp, #486673; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x12]6674; CHECK-SD-NEXT:    add x12, sp, #1766675; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x13]6676; CHECK-SD-NEXT:    add x13, sp, #6806677; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x10]6678; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x12]6679; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x13]6680; CHECK-SD-NEXT:    add x13, sp, #566681; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x9]6682; CHECK-SD-NEXT:    mov v1.b[2], w26683; CHECK-SD-NEXT:    add x8, sp, #1286684; CHECK-SD-NEXT:    add x14, sp, #1846685; CHECK-SD-NEXT:    add x11, sp, #1366686; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x13]6687; CHECK-SD-NEXT:    add x13, sp, #5526688; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x14]6689; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x13]6690; CHECK-SD-NEXT:    add x14, sp, #6246691; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x8]6692; CHECK-SD-NEXT:    add x8, sp, #6886693; CHECK-SD-NEXT:    add x13, sp, #5046694; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x14]6695; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]6696; CHECK-SD-NEXT:    add x8, sp, #5606697; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x13]6698; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x8]6699; CHECK-SD-NEXT:    mov v1.b[3], w36700; CHECK-SD-NEXT:    add x9, sp, #646701; CHECK-SD-NEXT:    add x15, sp, #6326702; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x9]6703; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x11]6704; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x15]6705; CHECK-SD-NEXT:    add x8, sp, #6966706; CHECK-SD-NEXT:    add x9, sp, #5686707; CHECK-SD-NEXT:    add x11, sp, #5126708; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x8]6709; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x9]6710; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x11]6711; CHECK-SD-NEXT:    add x8, sp, #6406712; CHECK-SD-NEXT:    mov v1.b[4], w46713; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]6714; CHECK-SD-NEXT:    add x8, sp, #7046715; CHECK-SD-NEXT:    add x9, sp, #5766716; CHECK-SD-NEXT:    add x11, sp, #5206717; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x8]6718; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x9]6719; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x11]6720; CHECK-SD-NEXT:    ldr b18, [sp, #736]6721; CHECK-SD-NEXT:    add x12, sp, #1926722; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x12]6723; CHECK-SD-NEXT:    add x8, sp, #6486724; CHECK-SD-NEXT:    add x9, sp, #5286725; CHECK-SD-NEXT:    add x11, sp, #7126726; CHECK-SD-NEXT:    add x12, sp, #5846727; CHECK-SD-NEXT:    sshll v18.8h, v18.8b, #06728; CHECK-SD-NEXT:    mov v1.b[5], w56729; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x11]6730; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x12]6731; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x8]6732; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x9]6733; CHECK-SD-NEXT:    movi v17.2d, #00000000000000006734; CHECK-SD-NEXT:    add x8, sp, #6566735; CHECK-SD-NEXT:    add x9, sp, #5366736; CHECK-SD-NEXT:    add x11, sp, #7206737; CHECK-SD-NEXT:    add x12, sp, #5926738; CHECK-SD-NEXT:    sshll v18.4s, v18.4h, #06739; CHECK-SD-NEXT:    ldr b7, [sp, #208]6740; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x11]6741; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x12]6742; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x8]6743; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x9]6744; CHECK-SD-NEXT:    mov v1.b[6], w66745; CHECK-SD-NEXT:    sshll v7.8h, v7.8b, #06746; CHECK-SD-NEXT:    add x8, sp, #6646747; CHECK-SD-NEXT:    add x9, sp, #7286748; CHECK-SD-NEXT:    add x11, sp, #6006749; CHECK-SD-NEXT:    mov v17.s[0], v18.s[0]6750; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x9]6751; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x11]6752; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x8]6753; CHECK-SD-NEXT:    sshll v5.8h, v5.8b, #06754; CHECK-SD-NEXT:    movi v18.2d, #00000000000000006755; CHECK-SD-NEXT:    add x10, sp, #2006756; CHECK-SD-NEXT:    mov v1.b[7], w76757; CHECK-SD-NEXT:    add x9, sp, #726758; CHECK-SD-NEXT:    sshll v7.4s, v7.4h, #06759; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x10]6760; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x9]6761; CHECK-SD-NEXT:    sshll v6.8h, v6.8b, #06762; CHECK-SD-NEXT:    sshll v16.8h, v16.8b, #06763; CHECK-SD-NEXT:    sshll v4.8h, v4.8b, #06764; CHECK-SD-NEXT:    saddw v17.4s, v17.4s, v5.4h6765; CHECK-SD-NEXT:    sshll v0.8h, v0.8b, #06766; CHECK-SD-NEXT:    mov v18.s[0], v7.s[0]6767; CHECK-SD-NEXT:    sshll v1.8h, v1.8b, #06768; CHECK-SD-NEXT:    sshll v2.8h, v2.8b, #06769; CHECK-SD-NEXT:    sshll v3.8h, v3.8b, #06770; CHECK-SD-NEXT:    saddl2 v7.4s, v16.8h, v6.8h6771; CHECK-SD-NEXT:    saddl2 v5.4s, v5.8h, v4.8h6772; CHECK-SD-NEXT:    saddl v6.4s, v16.4h, v6.4h6773; CHECK-SD-NEXT:    saddw v4.4s, v17.4s, v4.4h6774; CHECK-SD-NEXT:    saddl2 v17.4s, v1.8h, v0.8h6775; CHECK-SD-NEXT:    saddl2 v16.4s, v3.8h, v2.8h6776; CHECK-SD-NEXT:    saddw v1.4s, v18.4s, v1.4h6777; CHECK-SD-NEXT:    add v5.4s, v5.4s, v7.4s6778; CHECK-SD-NEXT:    add v4.4s, v4.4s, v6.4s6779; CHECK-SD-NEXT:    saddl v2.4s, v3.4h, v2.4h6780; CHECK-SD-NEXT:    add v6.4s, v17.4s, v16.4s6781; CHECK-SD-NEXT:    saddw v0.4s, v1.4s, v0.4h6782; CHECK-SD-NEXT:    add v1.4s, v4.4s, v5.4s6783; CHECK-SD-NEXT:    add v0.4s, v0.4s, v2.4s6784; CHECK-SD-NEXT:    add v1.4s, v6.4s, v1.4s6785; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s6786; CHECK-SD-NEXT:    addv s0, v0.4s6787; CHECK-SD-NEXT:    fmov w0, s06788; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload6789; CHECK-SD-NEXT:    ret6790;6791; CHECK-GI-LABEL: test_sdot_v33i8_double_nomla:6792; CHECK-GI:       // %bb.0: // %entry6793; CHECK-GI-NEXT:    stp d13, d12, [sp, #-64]! // 16-byte Folded Spill6794; CHECK-GI-NEXT:    stp d11, d10, [sp, #16] // 16-byte Folded Spill6795; CHECK-GI-NEXT:    stp d9, d8, [sp, #32] // 16-byte Folded Spill6796; CHECK-GI-NEXT:    str x29, [sp, #48] // 8-byte Spill6797; CHECK-GI-NEXT:    .cfi_def_cfa_offset 646798; CHECK-GI-NEXT:    .cfi_offset w29, -166799; CHECK-GI-NEXT:    .cfi_offset b8, -246800; CHECK-GI-NEXT:    .cfi_offset b9, -326801; CHECK-GI-NEXT:    .cfi_offset b10, -406802; CHECK-GI-NEXT:    .cfi_offset b11, -486803; CHECK-GI-NEXT:    .cfi_offset b12, -566804; CHECK-GI-NEXT:    .cfi_offset b13, -646805; CHECK-GI-NEXT:    sxtb w8, w06806; CHECK-GI-NEXT:    sxtb w9, w16807; CHECK-GI-NEXT:    sxtb w12, w46808; CHECK-GI-NEXT:    ldr w10, [sp, #72]6809; CHECK-GI-NEXT:    ldr w11, [sp, #96]6810; CHECK-GI-NEXT:    sxtb w13, w56811; CHECK-GI-NEXT:    fmov s22, w86812; CHECK-GI-NEXT:    ldr w8, [sp, #64]6813; CHECK-GI-NEXT:    fmov s23, w126814; CHECK-GI-NEXT:    sxtb w10, w106815; CHECK-GI-NEXT:    sxtb w12, w76816; CHECK-GI-NEXT:    movi d18, #00000000000000006817; CHECK-GI-NEXT:    sxtb w8, w86818; CHECK-GI-NEXT:    movi d19, #00000000000000006819; CHECK-GI-NEXT:    movi d20, #00000000000000006820; CHECK-GI-NEXT:    mov v22.s[1], w96821; CHECK-GI-NEXT:    sxtb w9, w26822; CHECK-GI-NEXT:    mov v23.s[1], w136823; CHECK-GI-NEXT:    fmov s24, w86824; CHECK-GI-NEXT:    sxtb w8, w116825; CHECK-GI-NEXT:    sxtb w11, w66826; CHECK-GI-NEXT:    ldr w13, [sp, #232]6827; CHECK-GI-NEXT:    mov v18.s[1], wzr6828; CHECK-GI-NEXT:    movi d21, #00000000000000006829; CHECK-GI-NEXT:    fmov s25, w86830; CHECK-GI-NEXT:    ldr w8, [sp, #80]6831; CHECK-GI-NEXT:    mov v19.s[1], wzr6832; CHECK-GI-NEXT:    mov v22.s[2], w96833; CHECK-GI-NEXT:    mov v24.s[1], w106834; CHECK-GI-NEXT:    sxtb w10, w36835; CHECK-GI-NEXT:    ldr w9, [sp, #104]6836; CHECK-GI-NEXT:    mov v23.s[2], w116837; CHECK-GI-NEXT:    sxtb w8, w86838; CHECK-GI-NEXT:    ldr w11, [sp, #136]6839; CHECK-GI-NEXT:    mov v18.s[2], wzr6840; CHECK-GI-NEXT:    movi d6, #00000000000000006841; CHECK-GI-NEXT:    sxtb w9, w96842; CHECK-GI-NEXT:    mov v19.s[2], wzr6843; CHECK-GI-NEXT:    movi d7, #00000000000000006844; CHECK-GI-NEXT:    mov v22.s[3], w106845; CHECK-GI-NEXT:    ldr w10, [sp, #128]6846; CHECK-GI-NEXT:    mov v24.s[2], w86847; CHECK-GI-NEXT:    mov v25.s[1], w96848; CHECK-GI-NEXT:    ldr w9, [sp, #112]6849; CHECK-GI-NEXT:    ldr w8, [sp, #88]6850; CHECK-GI-NEXT:    sxtb w10, w106851; CHECK-GI-NEXT:    sxtb w11, w116852; CHECK-GI-NEXT:    mov v23.s[3], w126853; CHECK-GI-NEXT:    sxtb w9, w96854; CHECK-GI-NEXT:    ldr w12, [sp, #160]6855; CHECK-GI-NEXT:    sxtb w8, w86856; CHECK-GI-NEXT:    fmov s26, w106857; CHECK-GI-NEXT:    ldr w10, [sp, #144]6858; CHECK-GI-NEXT:    movi d5, #00000000000000006859; CHECK-GI-NEXT:    mov v25.s[2], w96860; CHECK-GI-NEXT:    ldr w9, [sp, #120]6861; CHECK-GI-NEXT:    sxtb w12, w126862; CHECK-GI-NEXT:    mov v24.s[3], w86863; CHECK-GI-NEXT:    ldr w8, [sp, #168]6864; CHECK-GI-NEXT:    sxtb w10, w106865; CHECK-GI-NEXT:    mov v26.s[1], w116866; CHECK-GI-NEXT:    sxtb w9, w96867; CHECK-GI-NEXT:    fmov s27, w126868; CHECK-GI-NEXT:    sxtb w8, w86869; CHECK-GI-NEXT:    ldr w12, [sp, #224]6870; CHECK-GI-NEXT:    ldr w11, [sp, #152]6871; CHECK-GI-NEXT:    mov v25.s[3], w96872; CHECK-GI-NEXT:    ldr w9, [sp, #192]6873; CHECK-GI-NEXT:    add v22.4s, v22.4s, v23.4s6874; CHECK-GI-NEXT:    mov v27.s[1], w86875; CHECK-GI-NEXT:    movi d16, #00000000000000006876; CHECK-GI-NEXT:    movi d17, #00000000000000006877; CHECK-GI-NEXT:    mov v26.s[2], w106878; CHECK-GI-NEXT:    ldr w10, [sp, #200]6879; CHECK-GI-NEXT:    sxtb w9, w96880; CHECK-GI-NEXT:    movi d0, #00000000000000006881; CHECK-GI-NEXT:    movi d1, #00000000000000006882; CHECK-GI-NEXT:    movi d3, #00000000000000006883; CHECK-GI-NEXT:    sxtb w8, w106884; CHECK-GI-NEXT:    sxtb w10, w126885; CHECK-GI-NEXT:    fmov s28, w96886; CHECK-GI-NEXT:    ldr w9, [sp, #176]6887; CHECK-GI-NEXT:    sxtb w12, w136888; CHECK-GI-NEXT:    ldr w13, [sp, #208]6889; CHECK-GI-NEXT:    fmov s29, w106890; CHECK-GI-NEXT:    sxtb w10, w116891; CHECK-GI-NEXT:    ldr w11, [sp, #536]6892; CHECK-GI-NEXT:    sxtb w9, w96893; CHECK-GI-NEXT:    mov v28.s[1], w86894; CHECK-GI-NEXT:    ldr w8, [sp, #184]6895; CHECK-GI-NEXT:    mov v26.s[3], w106896; CHECK-GI-NEXT:    ldr w10, [sp, #528]6897; CHECK-GI-NEXT:    sxtb w13, w136898; CHECK-GI-NEXT:    mov v29.s[1], w126899; CHECK-GI-NEXT:    ldr w12, [sp, #240]6900; CHECK-GI-NEXT:    mov v27.s[2], w96901; CHECK-GI-NEXT:    sxtb w10, w106902; CHECK-GI-NEXT:    sxtb w8, w86903; CHECK-GI-NEXT:    ldr w9, [sp, #216]6904; CHECK-GI-NEXT:    sxtb w12, w126905; CHECK-GI-NEXT:    mov v28.s[2], w136906; CHECK-GI-NEXT:    sxtb w11, w116907; CHECK-GI-NEXT:    fmov s30, w106908; CHECK-GI-NEXT:    sxtb w9, w96909; CHECK-GI-NEXT:    ldr w10, [sp, #256]6910; CHECK-GI-NEXT:    mov v29.s[2], w126911; CHECK-GI-NEXT:    ldr w12, [sp, #248]6912; CHECK-GI-NEXT:    mov v27.s[3], w86913; CHECK-GI-NEXT:    ldr w8, [sp, #560]6914; CHECK-GI-NEXT:    sxtb w10, w106915; CHECK-GI-NEXT:    add v23.4s, v24.4s, v25.4s6916; CHECK-GI-NEXT:    sxtb w12, w126917; CHECK-GI-NEXT:    mov v30.s[1], w116918; CHECK-GI-NEXT:    ldr w11, [sp, #544]6919; CHECK-GI-NEXT:    sxtb w8, w86920; CHECK-GI-NEXT:    mov v28.s[3], w96921; CHECK-GI-NEXT:    ldr w9, [sp, #568]6922; CHECK-GI-NEXT:    mov v29.s[3], w126923; CHECK-GI-NEXT:    ldr w12, [sp, #592]6924; CHECK-GI-NEXT:    sxtb w11, w116925; CHECK-GI-NEXT:    fmov s31, w86926; CHECK-GI-NEXT:    sxtb w9, w96927; CHECK-GI-NEXT:    ldr w8, [sp, #552]6928; CHECK-GI-NEXT:    sxtb w12, w126929; CHECK-GI-NEXT:    mov v30.s[2], w116930; CHECK-GI-NEXT:    ldr w11, [sp, #600]6931; CHECK-GI-NEXT:    fmov s8, w106932; CHECK-GI-NEXT:    ldr w10, [sp, #624]6933; CHECK-GI-NEXT:    sxtb w8, w86934; CHECK-GI-NEXT:    mov v31.s[1], w96935; CHECK-GI-NEXT:    ldr w9, [sp, #576]6936; CHECK-GI-NEXT:    fmov s9, w126937; CHECK-GI-NEXT:    sxtb w11, w116938; CHECK-GI-NEXT:    sxtb w10, w106939; CHECK-GI-NEXT:    movi d2, #00000000000000006940; CHECK-GI-NEXT:    sxtb w9, w96941; CHECK-GI-NEXT:    mov v30.s[3], w86942; CHECK-GI-NEXT:    ldr w8, [sp, #632]6943; CHECK-GI-NEXT:    mov v9.s[1], w116944; CHECK-GI-NEXT:    ldr w11, [sp, #608]6945; CHECK-GI-NEXT:    fmov s10, w106946; CHECK-GI-NEXT:    mov v31.s[2], w96947; CHECK-GI-NEXT:    ldr w9, [sp, #656]6948; CHECK-GI-NEXT:    ldr w10, [sp, #688]6949; CHECK-GI-NEXT:    sxtb w11, w116950; CHECK-GI-NEXT:    sxtb w8, w86951; CHECK-GI-NEXT:    movi d4, #00000000000000006952; CHECK-GI-NEXT:    sxtb w9, w96953; CHECK-GI-NEXT:    sxtb w10, w106954; CHECK-GI-NEXT:    mov v18.s[3], wzr6955; CHECK-GI-NEXT:    mov v9.s[2], w116956; CHECK-GI-NEXT:    ldr w11, [sp, #664]6957; CHECK-GI-NEXT:    mov v10.s[1], w86958; CHECK-GI-NEXT:    ldr w8, [sp, #696]6959; CHECK-GI-NEXT:    fmov s11, w96960; CHECK-GI-NEXT:    ldr w9, [sp, #640]6961; CHECK-GI-NEXT:    fmov s12, w106962; CHECK-GI-NEXT:    sxtb w11, w116963; CHECK-GI-NEXT:    ldr w10, [sp, #672]6964; CHECK-GI-NEXT:    sxtb w8, w86965; CHECK-GI-NEXT:    sxtb w9, w96966; CHECK-GI-NEXT:    mov v19.s[3], wzr6967; CHECK-GI-NEXT:    mov v11.s[1], w116968; CHECK-GI-NEXT:    sxtb w10, w106969; CHECK-GI-NEXT:    mov v20.s[1], wzr6970; CHECK-GI-NEXT:    mov v12.s[1], w86971; CHECK-GI-NEXT:    ldr w8, [sp, #704]6972; CHECK-GI-NEXT:    mov v10.s[2], w96973; CHECK-GI-NEXT:    ldr w9, [sp, #648]6974; CHECK-GI-NEXT:    mov v21.s[1], wzr6975; CHECK-GI-NEXT:    mov v6.s[1], wzr6976; CHECK-GI-NEXT:    sxtb w8, w86977; CHECK-GI-NEXT:    mov v7.s[1], wzr6978; CHECK-GI-NEXT:    mov v5.s[1], wzr6979; CHECK-GI-NEXT:    sxtb w9, w96980; CHECK-GI-NEXT:    mov v11.s[2], w106981; CHECK-GI-NEXT:    ldr w10, [sp, #680]6982; CHECK-GI-NEXT:    mov v12.s[2], w86983; CHECK-GI-NEXT:    ldr w8, [sp, #712]6984; CHECK-GI-NEXT:    mov v16.s[1], wzr6985; CHECK-GI-NEXT:    mov v10.s[3], w96986; CHECK-GI-NEXT:    ldr w9, [sp, #720]6987; CHECK-GI-NEXT:    sxtb w10, w106988; CHECK-GI-NEXT:    sxtb w8, w86989; CHECK-GI-NEXT:    mov v17.s[1], wzr6990; CHECK-GI-NEXT:    add v19.4s, v18.4s, v19.4s6991; CHECK-GI-NEXT:    sxtb w9, w96992; CHECK-GI-NEXT:    mov v11.s[3], w106993; CHECK-GI-NEXT:    ldr w10, [sp, #728]6994; CHECK-GI-NEXT:    mov v12.s[3], w86995; CHECK-GI-NEXT:    ldr w8, [sp, #752]6996; CHECK-GI-NEXT:    mov v0.s[1], wzr6997; CHECK-GI-NEXT:    fmov s13, w96998; CHECK-GI-NEXT:    sxtb w10, w106999; CHECK-GI-NEXT:    ldr w9, [sp, #760]7000; CHECK-GI-NEXT:    sxtb w8, w87001; CHECK-GI-NEXT:    mov v1.s[1], wzr7002; CHECK-GI-NEXT:    mov v3.s[1], wzr7003; CHECK-GI-NEXT:    sxtb w9, w97004; CHECK-GI-NEXT:    mov v2.s[1], wzr7005; CHECK-GI-NEXT:    mov v4.s[1], wzr7006; CHECK-GI-NEXT:    mov v13.s[1], w107007; CHECK-GI-NEXT:    fmov s24, w87008; CHECK-GI-NEXT:    ldr w8, [sp, #736]7009; CHECK-GI-NEXT:    mov v20.s[2], wzr7010; CHECK-GI-NEXT:    mov v21.s[2], wzr7011; CHECK-GI-NEXT:    mov v6.s[2], wzr7012; CHECK-GI-NEXT:    sxtb w8, w87013; CHECK-GI-NEXT:    mov v7.s[2], wzr7014; CHECK-GI-NEXT:    mov v8.s[1], wzr7015; CHECK-GI-NEXT:    mov v24.s[1], w97016; CHECK-GI-NEXT:    ldr w9, [sp, #768]7017; CHECK-GI-NEXT:    ldr w12, [sp, #584]7018; CHECK-GI-NEXT:    mov v13.s[2], w87019; CHECK-GI-NEXT:    ldr w8, [sp, #784]7020; CHECK-GI-NEXT:    ldr w11, [sp, #616]7021; CHECK-GI-NEXT:    sxtb w9, w97022; CHECK-GI-NEXT:    mov v5.s[2], wzr7023; CHECK-GI-NEXT:    mov v16.s[2], wzr7024; CHECK-GI-NEXT:    sxtb w10, w87025; CHECK-GI-NEXT:    ldr w8, [sp, #744]7026; CHECK-GI-NEXT:    mov v17.s[2], wzr7027; CHECK-GI-NEXT:    mov v24.s[2], w97028; CHECK-GI-NEXT:    ldr w9, [sp, #776]7029; CHECK-GI-NEXT:    mov v0.s[2], wzr7030; CHECK-GI-NEXT:    fmov s18, w107031; CHECK-GI-NEXT:    mov v1.s[2], wzr7032; CHECK-GI-NEXT:    mov v3.s[2], wzr7033; CHECK-GI-NEXT:    mov v2.s[2], wzr7034; CHECK-GI-NEXT:    mov v4.s[2], wzr7035; CHECK-GI-NEXT:    mov v20.s[3], wzr7036; CHECK-GI-NEXT:    mov v21.s[3], wzr7037; CHECK-GI-NEXT:    mov v6.s[3], wzr7038; CHECK-GI-NEXT:    mov v7.s[3], wzr7039; CHECK-GI-NEXT:    mov v18.s[1], wzr7040; CHECK-GI-NEXT:    sxtb w12, w127041; CHECK-GI-NEXT:    sxtb w11, w117042; CHECK-GI-NEXT:    sxtb w8, w87043; CHECK-GI-NEXT:    sxtb w9, w97044; CHECK-GI-NEXT:    mov v8.s[2], wzr7045; CHECK-GI-NEXT:    mov v31.s[3], w127046; CHECK-GI-NEXT:    mov v9.s[3], w117047; CHECK-GI-NEXT:    mov v5.s[3], wzr7048; CHECK-GI-NEXT:    mov v16.s[3], wzr7049; CHECK-GI-NEXT:    mov v17.s[3], wzr7050; CHECK-GI-NEXT:    mov v13.s[3], w87051; CHECK-GI-NEXT:    mov v18.s[2], wzr7052; CHECK-GI-NEXT:    mov v24.s[3], w97053; CHECK-GI-NEXT:    mov v0.s[3], wzr7054; CHECK-GI-NEXT:    mov v1.s[3], wzr7055; CHECK-GI-NEXT:    mov v3.s[3], wzr7056; CHECK-GI-NEXT:    mov v2.s[3], wzr7057; CHECK-GI-NEXT:    mov v4.s[3], wzr7058; CHECK-GI-NEXT:    add v20.4s, v20.4s, v21.4s7059; CHECK-GI-NEXT:    add v6.4s, v6.4s, v7.4s7060; CHECK-GI-NEXT:    mov v8.s[3], wzr7061; CHECK-GI-NEXT:    add v25.4s, v26.4s, v27.4s7062; CHECK-GI-NEXT:    add v26.4s, v28.4s, v29.4s7063; CHECK-GI-NEXT:    mov v18.s[3], wzr7064; CHECK-GI-NEXT:    add v16.4s, v16.4s, v17.4s7065; CHECK-GI-NEXT:    add v7.4s, v30.4s, v31.4s7066; CHECK-GI-NEXT:    add v5.4s, v5.4s, v20.4s7067; CHECK-GI-NEXT:    add v17.4s, v9.4s, v10.4s7068; CHECK-GI-NEXT:    add v20.4s, v11.4s, v12.4s7069; CHECK-GI-NEXT:    add v21.4s, v13.4s, v24.4s7070; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s7071; CHECK-GI-NEXT:    add v2.4s, v2.4s, v4.4s7072; CHECK-GI-NEXT:    add v0.4s, v0.4s, v6.4s7073; CHECK-GI-NEXT:    add v3.4s, v22.4s, v23.4s7074; CHECK-GI-NEXT:    add v4.4s, v25.4s, v26.4s7075; CHECK-GI-NEXT:    add v5.4s, v8.4s, v5.4s7076; CHECK-GI-NEXT:    add v6.4s, v19.4s, v16.4s7077; CHECK-GI-NEXT:    add v7.4s, v7.4s, v17.4s7078; CHECK-GI-NEXT:    add v16.4s, v20.4s, v21.4s7079; CHECK-GI-NEXT:    add v1.4s, v1.4s, v2.4s7080; CHECK-GI-NEXT:    ldr x29, [sp, #48] // 8-byte Reload7081; CHECK-GI-NEXT:    add v0.4s, v18.4s, v0.4s7082; CHECK-GI-NEXT:    add v2.4s, v3.4s, v4.4s7083; CHECK-GI-NEXT:    add v3.4s, v5.4s, v6.4s7084; CHECK-GI-NEXT:    ldp d9, d8, [sp, #32] // 16-byte Folded Reload7085; CHECK-GI-NEXT:    add v4.4s, v7.4s, v16.4s7086; CHECK-GI-NEXT:    ldp d11, d10, [sp, #16] // 16-byte Folded Reload7087; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s7088; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s7089; CHECK-GI-NEXT:    add v0.4s, v4.4s, v0.4s7090; CHECK-GI-NEXT:    addv s1, v1.4s7091; CHECK-GI-NEXT:    addv s0, v0.4s7092; CHECK-GI-NEXT:    fmov w8, s17093; CHECK-GI-NEXT:    fmov w9, s07094; CHECK-GI-NEXT:    add w0, w8, w97095; CHECK-GI-NEXT:    ldp d13, d12, [sp], #64 // 16-byte Folded Reload7096; CHECK-GI-NEXT:    ret7097entry:7098  %az = sext <33 x i8> %a to <33 x i32>7099  %r1 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %az)7100  %cz = sext <33 x i8> %c to <33 x i32>7101  %r2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %cz)7102  %x = add i32 %r1, %r27103  ret i32 %x7104}7105 7106define i32 @test_udot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {7107; CHECK-SD-LABEL: test_udot_v48i8:7108; CHECK-SD:       // %bb.0: // %entry7109; CHECK-SD-NEXT:    movi v0.2d, #00000000000000007110; CHECK-SD-NEXT:    ldr q1, [x0, #32]7111; CHECK-SD-NEXT:    ldr q2, [x1, #32]7112; CHECK-SD-NEXT:    udot v0.4s, v2.16b, v1.16b7113; CHECK-SD-NEXT:    ldp q3, q1, [x0]7114; CHECK-SD-NEXT:    ldp q4, q2, [x1]7115; CHECK-SD-NEXT:    udot v0.4s, v4.16b, v3.16b7116; CHECK-SD-NEXT:    udot v0.4s, v2.16b, v1.16b7117; CHECK-SD-NEXT:    addv s0, v0.4s7118; CHECK-SD-NEXT:    fmov w8, s07119; CHECK-SD-NEXT:    add w0, w8, w27120; CHECK-SD-NEXT:    ret7121;7122; CHECK-GI-LABEL: test_udot_v48i8:7123; CHECK-GI:       // %bb.0: // %entry7124; CHECK-GI-NEXT:    movi d0, #00000000000000007125; CHECK-GI-NEXT:    movi v1.2d, #00000000000000007126; CHECK-GI-NEXT:    ldr q7, [x0, #32]7127; CHECK-GI-NEXT:    movi v2.2d, #00000000000000007128; CHECK-GI-NEXT:    movi v3.2d, #00000000000000007129; CHECK-GI-NEXT:    ldr q17, [x1, #32]7130; CHECK-GI-NEXT:    ldp q4, q5, [x0]7131; CHECK-GI-NEXT:    ldp q6, q16, [x1]7132; CHECK-GI-NEXT:    mov v0.s[1], wzr7133; CHECK-GI-NEXT:    udot v2.4s, v17.16b, v7.16b7134; CHECK-GI-NEXT:    udot v1.4s, v6.16b, v4.16b7135; CHECK-GI-NEXT:    udot v3.4s, v16.16b, v5.16b7136; CHECK-GI-NEXT:    mov v0.s[2], wzr7137; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s7138; CHECK-GI-NEXT:    mov v0.s[3], wzr7139; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s7140; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s7141; CHECK-GI-NEXT:    addv s0, v0.4s7142; CHECK-GI-NEXT:    fmov w8, s07143; CHECK-GI-NEXT:    add w0, w8, w27144; CHECK-GI-NEXT:    ret7145entry:7146  %0 = load <48 x i8>, ptr %a7147  %1 = zext <48 x i8> %0 to <48 x i32>7148  %2 = load <48 x i8>, ptr %b7149  %3 = zext <48 x i8> %2 to <48 x i32>7150  %4 = mul nuw nsw <48 x i32> %3, %17151  %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4)7152  %op.extra = add i32 %5, %sum7153  ret i32 %op.extra7154}7155 7156define i32 @test_udot_v48i8_nomla(ptr nocapture readonly %a1) {7157; CHECK-SD-LABEL: test_udot_v48i8_nomla:7158; CHECK-SD:       // %bb.0: // %entry7159; CHECK-SD-NEXT:    movi v0.16b, #17160; CHECK-SD-NEXT:    movi v1.2d, #00000000000000007161; CHECK-SD-NEXT:    ldr q2, [x0, #32]7162; CHECK-SD-NEXT:    udot v1.4s, v2.16b, v0.16b7163; CHECK-SD-NEXT:    ldp q3, q2, [x0]7164; CHECK-SD-NEXT:    udot v1.4s, v3.16b, v0.16b7165; CHECK-SD-NEXT:    udot v1.4s, v2.16b, v0.16b7166; CHECK-SD-NEXT:    addv s0, v1.4s7167; CHECK-SD-NEXT:    fmov w0, s07168; CHECK-SD-NEXT:    ret7169;7170; CHECK-GI-LABEL: test_udot_v48i8_nomla:7171; CHECK-GI:       // %bb.0: // %entry7172; CHECK-GI-NEXT:    movi d0, #00000000000000007173; CHECK-GI-NEXT:    movi v1.16b, #17174; CHECK-GI-NEXT:    ldr q7, [x0, #32]7175; CHECK-GI-NEXT:    movi v2.2d, #00000000000000007176; CHECK-GI-NEXT:    movi v3.2d, #00000000000000007177; CHECK-GI-NEXT:    movi v4.2d, #00000000000000007178; CHECK-GI-NEXT:    ldp q5, q6, [x0]7179; CHECK-GI-NEXT:    mov v0.s[1], wzr7180; CHECK-GI-NEXT:    udot v2.4s, v5.16b, v1.16b7181; CHECK-GI-NEXT:    udot v4.4s, v6.16b, v1.16b7182; CHECK-GI-NEXT:    udot v3.4s, v7.16b, v1.16b7183; CHECK-GI-NEXT:    mov v0.s[2], wzr7184; CHECK-GI-NEXT:    add v1.4s, v2.4s, v4.4s7185; CHECK-GI-NEXT:    mov v0.s[3], wzr7186; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s7187; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s7188; CHECK-GI-NEXT:    addv s0, v0.4s7189; CHECK-GI-NEXT:    fmov w0, s07190; CHECK-GI-NEXT:    ret7191entry:7192  %0 = load <48 x i8>, ptr %a17193  %1 = zext <48 x i8> %0 to <48 x i32>7194  %2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %1)7195  ret i32 %27196}7197define i32 @test_sdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {7198; CHECK-SD-LABEL: test_sdot_v48i8:7199; CHECK-SD:       // %bb.0: // %entry7200; CHECK-SD-NEXT:    movi v0.2d, #00000000000000007201; CHECK-SD-NEXT:    ldr q1, [x0, #32]7202; CHECK-SD-NEXT:    ldr q2, [x1, #32]7203; CHECK-SD-NEXT:    sdot v0.4s, v2.16b, v1.16b7204; CHECK-SD-NEXT:    ldp q3, q1, [x0]7205; CHECK-SD-NEXT:    ldp q4, q2, [x1]7206; CHECK-SD-NEXT:    sdot v0.4s, v4.16b, v3.16b7207; CHECK-SD-NEXT:    sdot v0.4s, v2.16b, v1.16b7208; CHECK-SD-NEXT:    addv s0, v0.4s7209; CHECK-SD-NEXT:    fmov w8, s07210; CHECK-SD-NEXT:    add w0, w8, w27211; CHECK-SD-NEXT:    ret7212;7213; CHECK-GI-LABEL: test_sdot_v48i8:7214; CHECK-GI:       // %bb.0: // %entry7215; CHECK-GI-NEXT:    movi d0, #00000000000000007216; CHECK-GI-NEXT:    movi v1.2d, #00000000000000007217; CHECK-GI-NEXT:    ldr q7, [x0, #32]7218; CHECK-GI-NEXT:    movi v2.2d, #00000000000000007219; CHECK-GI-NEXT:    movi v3.2d, #00000000000000007220; CHECK-GI-NEXT:    ldr q17, [x1, #32]7221; CHECK-GI-NEXT:    ldp q4, q5, [x0]7222; CHECK-GI-NEXT:    ldp q6, q16, [x1]7223; CHECK-GI-NEXT:    mov v0.s[1], wzr7224; CHECK-GI-NEXT:    sdot v2.4s, v17.16b, v7.16b7225; CHECK-GI-NEXT:    sdot v1.4s, v6.16b, v4.16b7226; CHECK-GI-NEXT:    sdot v3.4s, v16.16b, v5.16b7227; CHECK-GI-NEXT:    mov v0.s[2], wzr7228; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s7229; CHECK-GI-NEXT:    mov v0.s[3], wzr7230; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s7231; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s7232; CHECK-GI-NEXT:    addv s0, v0.4s7233; CHECK-GI-NEXT:    fmov w8, s07234; CHECK-GI-NEXT:    add w0, w8, w27235; CHECK-GI-NEXT:    ret7236entry:7237  %0 = load <48 x i8>, ptr %a7238  %1 = sext <48 x i8> %0 to <48 x i32>7239  %2 = load <48 x i8>, ptr %b7240  %3 = sext <48 x i8> %2 to <48 x i32>7241  %4 = mul nsw <48 x i32> %3, %17242  %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4)7243  %op.extra = add nsw i32 %5, %sum7244  ret i32 %op.extra7245}7246 7247define i32 @test_sdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {7248; CHECK-SD-LABEL: test_sdot_v48i8_double:7249; CHECK-SD:       // %bb.0: // %entry7250; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill7251; CHECK-SD-NEXT:    .cfi_def_cfa_offset 167252; CHECK-SD-NEXT:    .cfi_offset w29, -167253; CHECK-SD-NEXT:    ldr b3, [sp, #592]7254; CHECK-SD-NEXT:    add x8, sp, #6007255; CHECK-SD-NEXT:    ldr b6, [sp, #208]7256; CHECK-SD-NEXT:    ldr b0, [sp, #336]7257; CHECK-SD-NEXT:    add x9, sp, #3447258; CHECK-SD-NEXT:    ldr b2, [sp, #464]7259; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x8]7260; CHECK-SD-NEXT:    add x8, sp, #2167261; CHECK-SD-NEXT:    add x10, sp, #6247262; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x8]7263; CHECK-SD-NEXT:    add x8, sp, #6087264; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x9]7265; CHECK-SD-NEXT:    add x9, sp, #2327266; CHECK-SD-NEXT:    fmov s1, w07267; CHECK-SD-NEXT:    ldr b7, [sp, #1360]7268; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x8]7269; CHECK-SD-NEXT:    add x8, sp, #2247270; CHECK-SD-NEXT:    add x11, sp, #6487271; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]7272; CHECK-SD-NEXT:    add x8, sp, #6167273; CHECK-SD-NEXT:    add x12, sp, #3767274; CHECK-SD-NEXT:    mov v1.b[1], w17275; CHECK-SD-NEXT:    ldr b16, [sp, #976]7276; CHECK-SD-NEXT:    add x14, sp, #2887277; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x8]7278; CHECK-SD-NEXT:    add x8, sp, #6327279; CHECK-SD-NEXT:    add x15, sp, #4087280; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x9]7281; CHECK-SD-NEXT:    add x9, sp, #4727282; CHECK-SD-NEXT:    add x13, sp, #6967283; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x9]7284; CHECK-SD-NEXT:    add x9, sp, #2407285; CHECK-SD-NEXT:    add x16, sp, #4487286; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x10]7287; CHECK-SD-NEXT:    add x10, sp, #3527288; CHECK-SD-NEXT:    mov v1.b[2], w27289; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x9]7290; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x10]7291; CHECK-SD-NEXT:    add x10, sp, #13687292; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x10]7293; CHECK-SD-NEXT:    add x10, sp, #2487294; CHECK-SD-NEXT:    add x9, sp, #6407295; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x8]7296; CHECK-SD-NEXT:    add x8, sp, #6567297; CHECK-SD-NEXT:    movi v5.2d, #00000000000000007298; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x10]7299; CHECK-SD-NEXT:    add x10, sp, #3607300; CHECK-SD-NEXT:    mov v1.b[3], w37301; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x10]7302; CHECK-SD-NEXT:    add x10, sp, #2567303; CHECK-SD-NEXT:    movi v4.2d, #00000000000000007304; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x9]7305; CHECK-SD-NEXT:    add x9, sp, #3687306; CHECK-SD-NEXT:    ldr b17, [sp, #720]7307; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x10]7308; CHECK-SD-NEXT:    add x10, sp, #9847309; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x9]7310; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x10]7311; CHECK-SD-NEXT:    add x10, sp, #6647312; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x11]7313; CHECK-SD-NEXT:    add x11, sp, #2647314; CHECK-SD-NEXT:    mov v1.b[4], w47315; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x11]7316; CHECK-SD-NEXT:    add x9, sp, #6727317; CHECK-SD-NEXT:    add x11, sp, #6807318; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x12]7319; CHECK-SD-NEXT:    add x12, sp, #4807320; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x12]7321; CHECK-SD-NEXT:    add x12, sp, #2727322; CHECK-SD-NEXT:    ld1 { v3.b }[8], [x8]7323; CHECK-SD-NEXT:    ld1 { v6.b }[8], [x12]7324; CHECK-SD-NEXT:    add x12, sp, #3847325; CHECK-SD-NEXT:    mov v1.b[5], w57326; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x12]7327; CHECK-SD-NEXT:    add x12, sp, #2807328; CHECK-SD-NEXT:    add x8, sp, #6887329; CHECK-SD-NEXT:    ld1 { v3.b }[9], [x10]7330; CHECK-SD-NEXT:    add x10, sp, #13767331; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x10]7332; CHECK-SD-NEXT:    add x10, sp, #3927333; CHECK-SD-NEXT:    ld1 { v6.b }[9], [x12]7334; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x10]7335; CHECK-SD-NEXT:    mov v1.b[6], w67336; CHECK-SD-NEXT:    add x12, sp, #7047337; CHECK-SD-NEXT:    ld1 { v3.b }[10], [x9]7338; CHECK-SD-NEXT:    add x9, sp, #4007339; CHECK-SD-NEXT:    add x10, sp, #7127340; CHECK-SD-NEXT:    ld1 { v6.b }[10], [x14]7341; CHECK-SD-NEXT:    add x14, sp, #9927342; CHECK-SD-NEXT:    ld1 { v0.b }[8], [x9]7343; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x14]7344; CHECK-SD-NEXT:    add x14, sp, #2967345; CHECK-SD-NEXT:    ld1 { v3.b }[11], [x11]7346; CHECK-SD-NEXT:    add x9, sp, #3047347; CHECK-SD-NEXT:    add x11, sp, #3127348; CHECK-SD-NEXT:    ld1 { v6.b }[11], [x14]7349; CHECK-SD-NEXT:    mov v1.b[7], w77350; CHECK-SD-NEXT:    add x14, sp, #3207351; CHECK-SD-NEXT:    ld1 { v0.b }[9], [x15]7352; CHECK-SD-NEXT:    add x15, sp, #3287353; CHECK-SD-NEXT:    ld1 { v3.b }[12], [x8]7354; CHECK-SD-NEXT:    add x8, sp, #4167355; CHECK-SD-NEXT:    ld1 { v6.b }[12], [x9]7356; CHECK-SD-NEXT:    add x9, sp, #13847357; CHECK-SD-NEXT:    ld1 { v0.b }[10], [x8]7358; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]7359; CHECK-SD-NEXT:    add x9, sp, #4247360; CHECK-SD-NEXT:    ld1 { v3.b }[13], [x13]7361; CHECK-SD-NEXT:    add x8, sp, #4327362; CHECK-SD-NEXT:    add x13, sp, #4407363; CHECK-SD-NEXT:    ld1 { v6.b }[13], [x11]7364; CHECK-SD-NEXT:    add x11, sp, #167365; CHECK-SD-NEXT:    ld1 { v0.b }[11], [x9]7366; CHECK-SD-NEXT:    add x9, sp, #10007367; CHECK-SD-NEXT:    ld1 { v1.b }[8], [x11]7368; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x9]7369; CHECK-SD-NEXT:    ld1 { v3.b }[14], [x12]7370; CHECK-SD-NEXT:    add x12, sp, #4887371; CHECK-SD-NEXT:    ld1 { v6.b }[14], [x14]7372; CHECK-SD-NEXT:    add x14, sp, #13927373; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x12]7374; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x14]7375; CHECK-SD-NEXT:    add x11, sp, #10087376; CHECK-SD-NEXT:    ld1 { v0.b }[12], [x8]7377; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x11]7378; CHECK-SD-NEXT:    add x8, sp, #14007379; CHECK-SD-NEXT:    ld1 { v3.b }[15], [x10]7380; CHECK-SD-NEXT:    add x10, sp, #4967381; CHECK-SD-NEXT:    add x9, sp, #247382; CHECK-SD-NEXT:    ld1 { v6.b }[15], [x15]7383; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x8]7384; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x10]7385; CHECK-SD-NEXT:    add x10, sp, #10167386; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x10]7387; CHECK-SD-NEXT:    ld1 { v0.b }[13], [x13]7388; CHECK-SD-NEXT:    add x8, sp, #14087389; CHECK-SD-NEXT:    ld1 { v1.b }[9], [x9]7390; CHECK-SD-NEXT:    add x9, sp, #5047391; CHECK-SD-NEXT:    add x10, sp, #5127392; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x8]7393; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x9]7394; CHECK-SD-NEXT:    add x9, sp, #10247395; CHECK-SD-NEXT:    add x8, sp, #327396; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x9]7397; CHECK-SD-NEXT:    ld1 { v0.b }[14], [x16]7398; CHECK-SD-NEXT:    ld1 { v1.b }[10], [x8]7399; CHECK-SD-NEXT:    add x8, sp, #14167400; CHECK-SD-NEXT:    add x9, sp, #4567401; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x8]7402; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x10]7403; CHECK-SD-NEXT:    add x10, sp, #10327404; CHECK-SD-NEXT:    add x8, sp, #407405; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x10]7406; CHECK-SD-NEXT:    ld1 { v0.b }[15], [x9]7407; CHECK-SD-NEXT:    ld1 { v1.b }[11], [x8]7408; CHECK-SD-NEXT:    add x8, sp, #14247409; CHECK-SD-NEXT:    add x9, sp, #5207410; CHECK-SD-NEXT:    ld1 { v7.b }[8], [x8]7411; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x9]7412; CHECK-SD-NEXT:    add x9, sp, #10407413; CHECK-SD-NEXT:    add x8, sp, #487414; CHECK-SD-NEXT:    ld1 { v16.b }[8], [x9]7415; CHECK-SD-NEXT:    add x10, sp, #5287416; CHECK-SD-NEXT:    ld1 { v1.b }[12], [x8]7417; CHECK-SD-NEXT:    add x8, sp, #14327418; CHECK-SD-NEXT:    sdot v5.4s, v6.16b, v3.16b7419; CHECK-SD-NEXT:    ld1 { v7.b }[9], [x8]7420; CHECK-SD-NEXT:    ld1 { v2.b }[8], [x10]7421; CHECK-SD-NEXT:    add x8, sp, #10487422; CHECK-SD-NEXT:    ldr b3, [sp, #80]7423; CHECK-SD-NEXT:    ld1 { v16.b }[9], [x8]7424; CHECK-SD-NEXT:    add x10, sp, #887425; CHECK-SD-NEXT:    add x8, sp, #5367426; CHECK-SD-NEXT:    add x11, sp, #14407427; CHECK-SD-NEXT:    add x9, sp, #567428; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x10]7429; CHECK-SD-NEXT:    ld1 { v2.b }[9], [x8]7430; CHECK-SD-NEXT:    add x8, sp, #10567431; CHECK-SD-NEXT:    ld1 { v7.b }[10], [x11]7432; CHECK-SD-NEXT:    ld1 { v16.b }[10], [x8]7433; CHECK-SD-NEXT:    ld1 { v1.b }[13], [x9]7434; CHECK-SD-NEXT:    add x9, sp, #967435; CHECK-SD-NEXT:    add x8, sp, #5447436; CHECK-SD-NEXT:    add x10, sp, #14487437; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x9]7438; CHECK-SD-NEXT:    ld1 { v2.b }[10], [x8]7439; CHECK-SD-NEXT:    add x8, sp, #10647440; CHECK-SD-NEXT:    ld1 { v7.b }[11], [x10]7441; CHECK-SD-NEXT:    ld1 { v16.b }[11], [x8]7442; CHECK-SD-NEXT:    add x10, sp, #1047443; CHECK-SD-NEXT:    add x8, sp, #5527444; CHECK-SD-NEXT:    add x11, sp, #14567445; CHECK-SD-NEXT:    add x9, sp, #647446; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x10]7447; CHECK-SD-NEXT:    ld1 { v2.b }[11], [x8]7448; CHECK-SD-NEXT:    add x8, sp, #10727449; CHECK-SD-NEXT:    ld1 { v7.b }[12], [x11]7450; CHECK-SD-NEXT:    ld1 { v16.b }[12], [x8]7451; CHECK-SD-NEXT:    ld1 { v1.b }[14], [x9]7452; CHECK-SD-NEXT:    add x9, sp, #1127453; CHECK-SD-NEXT:    add x8, sp, #5607454; CHECK-SD-NEXT:    add x10, sp, #14647455; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x9]7456; CHECK-SD-NEXT:    ld1 { v2.b }[12], [x8]7457; CHECK-SD-NEXT:    add x8, sp, #10807458; CHECK-SD-NEXT:    ld1 { v7.b }[13], [x10]7459; CHECK-SD-NEXT:    ld1 { v16.b }[13], [x8]7460; CHECK-SD-NEXT:    add x10, sp, #1207461; CHECK-SD-NEXT:    add x8, sp, #5687462; CHECK-SD-NEXT:    add x11, sp, #14727463; CHECK-SD-NEXT:    add x9, sp, #727464; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x10]7465; CHECK-SD-NEXT:    ld1 { v2.b }[13], [x8]7466; CHECK-SD-NEXT:    add x8, sp, #10887467; CHECK-SD-NEXT:    ld1 { v7.b }[14], [x11]7468; CHECK-SD-NEXT:    ld1 { v16.b }[14], [x8]7469; CHECK-SD-NEXT:    ld1 { v1.b }[15], [x9]7470; CHECK-SD-NEXT:    add x9, sp, #1287471; CHECK-SD-NEXT:    ldr b6, [sp, #1104]7472; CHECK-SD-NEXT:    add x10, sp, #14807473; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x9]7474; CHECK-SD-NEXT:    add x8, sp, #10967475; CHECK-SD-NEXT:    add x9, sp, #11127476; CHECK-SD-NEXT:    ld1 { v7.b }[15], [x10]7477; CHECK-SD-NEXT:    ld1 { v16.b }[15], [x8]7478; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x9]7479; CHECK-SD-NEXT:    add x8, sp, #7287480; CHECK-SD-NEXT:    add x9, sp, #5767481; CHECK-SD-NEXT:    add x10, sp, #1367482; CHECK-SD-NEXT:    ld1 { v17.b }[1], [x8]7483; CHECK-SD-NEXT:    add x8, sp, #11207484; CHECK-SD-NEXT:    ld1 { v2.b }[14], [x9]7485; CHECK-SD-NEXT:    sdot v4.4s, v16.16b, v7.16b7486; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]7487; CHECK-SD-NEXT:    add x8, sp, #7367488; CHECK-SD-NEXT:    ldr b7, [sp, #1232]7489; CHECK-SD-NEXT:    ldr b16, [sp, #848]7490; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x10]7491; CHECK-SD-NEXT:    ld1 { v17.b }[2], [x8]7492; CHECK-SD-NEXT:    add x9, sp, #12407493; CHECK-SD-NEXT:    add x10, sp, #8567494; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x9]7495; CHECK-SD-NEXT:    ld1 { v16.b }[1], [x10]7496; CHECK-SD-NEXT:    add x8, sp, #11287497; CHECK-SD-NEXT:    add x11, sp, #7447498; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x8]7499; CHECK-SD-NEXT:    add x10, sp, #12487500; CHECK-SD-NEXT:    ld1 { v17.b }[3], [x11]7501; CHECK-SD-NEXT:    add x11, sp, #8647502; CHECK-SD-NEXT:    add x9, sp, #1447503; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x10]7504; CHECK-SD-NEXT:    ld1 { v16.b }[2], [x11]7505; CHECK-SD-NEXT:    add x8, sp, #11367506; CHECK-SD-NEXT:    add x12, sp, #7527507; CHECK-SD-NEXT:    ld1 { v3.b }[8], [x9]7508; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x8]7509; CHECK-SD-NEXT:    ld1 { v17.b }[4], [x12]7510; CHECK-SD-NEXT:    add x9, sp, #12567511; CHECK-SD-NEXT:    add x10, sp, #8727512; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x9]7513; CHECK-SD-NEXT:    ld1 { v16.b }[3], [x10]7514; CHECK-SD-NEXT:    add x8, sp, #11447515; CHECK-SD-NEXT:    add x11, sp, #7607516; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x8]7517; CHECK-SD-NEXT:    add x10, sp, #12647518; CHECK-SD-NEXT:    ld1 { v17.b }[5], [x11]7519; CHECK-SD-NEXT:    add x11, sp, #8807520; CHECK-SD-NEXT:    add x9, sp, #1527521; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x10]7522; CHECK-SD-NEXT:    ld1 { v16.b }[4], [x11]7523; CHECK-SD-NEXT:    add x8, sp, #11527524; CHECK-SD-NEXT:    add x12, sp, #7687525; CHECK-SD-NEXT:    ld1 { v3.b }[9], [x9]7526; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x8]7527; CHECK-SD-NEXT:    ld1 { v17.b }[6], [x12]7528; CHECK-SD-NEXT:    add x9, sp, #12727529; CHECK-SD-NEXT:    add x10, sp, #8887530; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x9]7531; CHECK-SD-NEXT:    ld1 { v16.b }[5], [x10]7532; CHECK-SD-NEXT:    add x8, sp, #11607533; CHECK-SD-NEXT:    add x11, sp, #7767534; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x8]7535; CHECK-SD-NEXT:    add x10, sp, #12807536; CHECK-SD-NEXT:    ld1 { v17.b }[7], [x11]7537; CHECK-SD-NEXT:    add x11, sp, #8967538; CHECK-SD-NEXT:    add x9, sp, #1607539; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x10]7540; CHECK-SD-NEXT:    ld1 { v16.b }[6], [x11]7541; CHECK-SD-NEXT:    add x8, sp, #11687542; CHECK-SD-NEXT:    add x12, sp, #7847543; CHECK-SD-NEXT:    ld1 { v3.b }[10], [x9]7544; CHECK-SD-NEXT:    ld1 { v6.b }[8], [x8]7545; CHECK-SD-NEXT:    ld1 { v17.b }[8], [x12]7546; CHECK-SD-NEXT:    add x9, sp, #12887547; CHECK-SD-NEXT:    add x10, sp, #9047548; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x9]7549; CHECK-SD-NEXT:    ld1 { v16.b }[7], [x10]7550; CHECK-SD-NEXT:    add x8, sp, #11767551; CHECK-SD-NEXT:    add x11, sp, #7927552; CHECK-SD-NEXT:    ld1 { v6.b }[9], [x8]7553; CHECK-SD-NEXT:    add x10, sp, #12967554; CHECK-SD-NEXT:    ld1 { v17.b }[9], [x11]7555; CHECK-SD-NEXT:    add x11, sp, #9127556; CHECK-SD-NEXT:    add x9, sp, #1687557; CHECK-SD-NEXT:    ld1 { v7.b }[8], [x10]7558; CHECK-SD-NEXT:    ld1 { v16.b }[8], [x11]7559; CHECK-SD-NEXT:    add x8, sp, #11847560; CHECK-SD-NEXT:    add x12, sp, #8007561; CHECK-SD-NEXT:    ld1 { v3.b }[11], [x9]7562; CHECK-SD-NEXT:    ld1 { v6.b }[10], [x8]7563; CHECK-SD-NEXT:    ld1 { v17.b }[10], [x12]7564; CHECK-SD-NEXT:    add x9, sp, #13047565; CHECK-SD-NEXT:    add x10, sp, #9207566; CHECK-SD-NEXT:    ld1 { v7.b }[9], [x9]7567; CHECK-SD-NEXT:    ld1 { v16.b }[9], [x10]7568; CHECK-SD-NEXT:    add x8, sp, #11927569; CHECK-SD-NEXT:    add x11, sp, #8087570; CHECK-SD-NEXT:    ld1 { v6.b }[11], [x8]7571; CHECK-SD-NEXT:    add x10, sp, #13127572; CHECK-SD-NEXT:    ld1 { v17.b }[11], [x11]7573; CHECK-SD-NEXT:    add x11, sp, #9287574; CHECK-SD-NEXT:    add x9, sp, #1767575; CHECK-SD-NEXT:    ld1 { v7.b }[10], [x10]7576; CHECK-SD-NEXT:    ld1 { v16.b }[10], [x11]7577; CHECK-SD-NEXT:    add x8, sp, #12007578; CHECK-SD-NEXT:    add x12, sp, #8167579; CHECK-SD-NEXT:    ld1 { v3.b }[12], [x9]7580; CHECK-SD-NEXT:    ld1 { v6.b }[12], [x8]7581; CHECK-SD-NEXT:    ld1 { v17.b }[12], [x12]7582; CHECK-SD-NEXT:    add x9, sp, #13207583; CHECK-SD-NEXT:    add x10, sp, #9367584; CHECK-SD-NEXT:    ld1 { v7.b }[11], [x9]7585; CHECK-SD-NEXT:    ld1 { v16.b }[11], [x10]7586; CHECK-SD-NEXT:    add x8, sp, #12087587; CHECK-SD-NEXT:    add x11, sp, #8247588; CHECK-SD-NEXT:    ld1 { v6.b }[13], [x8]7589; CHECK-SD-NEXT:    add x10, sp, #13287590; CHECK-SD-NEXT:    ld1 { v17.b }[13], [x11]7591; CHECK-SD-NEXT:    add x11, sp, #9447592; CHECK-SD-NEXT:    add x9, sp, #1847593; CHECK-SD-NEXT:    ld1 { v7.b }[12], [x10]7594; CHECK-SD-NEXT:    ld1 { v16.b }[12], [x11]7595; CHECK-SD-NEXT:    add x8, sp, #12167596; CHECK-SD-NEXT:    add x12, sp, #8327597; CHECK-SD-NEXT:    ld1 { v3.b }[13], [x9]7598; CHECK-SD-NEXT:    ld1 { v6.b }[14], [x8]7599; CHECK-SD-NEXT:    ld1 { v17.b }[14], [x12]7600; CHECK-SD-NEXT:    add x9, sp, #13367601; CHECK-SD-NEXT:    add x10, sp, #9527602; CHECK-SD-NEXT:    ld1 { v7.b }[13], [x9]7603; CHECK-SD-NEXT:    ld1 { v16.b }[13], [x10]7604; CHECK-SD-NEXT:    add x8, sp, #12247605; CHECK-SD-NEXT:    add x11, sp, #8407606; CHECK-SD-NEXT:    ld1 { v6.b }[15], [x8]7607; CHECK-SD-NEXT:    add x8, sp, #1927608; CHECK-SD-NEXT:    ld1 { v17.b }[15], [x11]7609; CHECK-SD-NEXT:    add x10, sp, #13447610; CHECK-SD-NEXT:    add x11, sp, #9607611; CHECK-SD-NEXT:    ld1 { v3.b }[14], [x8]7612; CHECK-SD-NEXT:    ld1 { v7.b }[14], [x10]7613; CHECK-SD-NEXT:    ld1 { v16.b }[14], [x11]7614; CHECK-SD-NEXT:    add x9, sp, #5847615; CHECK-SD-NEXT:    sdot v5.4s, v1.16b, v0.16b7616; CHECK-SD-NEXT:    add x8, sp, #2007617; CHECK-SD-NEXT:    sdot v4.4s, v17.16b, v6.16b7618; CHECK-SD-NEXT:    ld1 { v2.b }[15], [x9]7619; CHECK-SD-NEXT:    add x9, sp, #13527620; CHECK-SD-NEXT:    add x10, sp, #9687621; CHECK-SD-NEXT:    ld1 { v3.b }[15], [x8]7622; CHECK-SD-NEXT:    ld1 { v7.b }[15], [x9]7623; CHECK-SD-NEXT:    ld1 { v16.b }[15], [x10]7624; CHECK-SD-NEXT:    sdot v5.4s, v3.16b, v2.16b7625; CHECK-SD-NEXT:    sdot v4.4s, v16.16b, v7.16b7626; CHECK-SD-NEXT:    add v0.4s, v5.4s, v4.4s7627; CHECK-SD-NEXT:    addv s0, v0.4s7628; CHECK-SD-NEXT:    fmov w0, s07629; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload7630; CHECK-SD-NEXT:    ret7631;7632; CHECK-GI-LABEL: test_sdot_v48i8_double:7633; CHECK-GI:       // %bb.0: // %entry7634; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill7635; CHECK-GI-NEXT:    .cfi_def_cfa_offset 167636; CHECK-GI-NEXT:    .cfi_offset w29, -167637; CHECK-GI-NEXT:    ldr w10, [sp, #80]7638; CHECK-GI-NEXT:    ldr w9, [sp, #88]7639; CHECK-GI-NEXT:    fmov s2, w07640; CHECK-GI-NEXT:    ldr w11, [sp, #208]7641; CHECK-GI-NEXT:    ldr w8, [sp, #216]7642; CHECK-GI-NEXT:    movi d1, #00000000000000007643; CHECK-GI-NEXT:    fmov s3, w107644; CHECK-GI-NEXT:    ldr w10, [sp, #336]7645; CHECK-GI-NEXT:    ldr w12, [sp, #720]7646; CHECK-GI-NEXT:    fmov s4, w117647; CHECK-GI-NEXT:    mov v2.b[1], w17648; CHECK-GI-NEXT:    ldr w11, [sp, #344]7649; CHECK-GI-NEXT:    fmov s5, w107650; CHECK-GI-NEXT:    ldr w10, [sp, #592]7651; CHECK-GI-NEXT:    fmov s16, w127652; CHECK-GI-NEXT:    mov v3.b[1], w97653; CHECK-GI-NEXT:    ldr w9, [sp, #464]7654; CHECK-GI-NEXT:    ldr w12, [sp, #1112]7655; CHECK-GI-NEXT:    fmov s7, w107656; CHECK-GI-NEXT:    mov v4.b[1], w87657; CHECK-GI-NEXT:    ldr w8, [sp, #472]7658; CHECK-GI-NEXT:    fmov s6, w97659; CHECK-GI-NEXT:    ldr w9, [sp, #600]7660; CHECK-GI-NEXT:    mov v5.b[1], w117661; CHECK-GI-NEXT:    ldr w10, [sp, #224]7662; CHECK-GI-NEXT:    mov v2.b[2], w27663; CHECK-GI-NEXT:    ldr w11, [sp, #16]7664; CHECK-GI-NEXT:    mov v7.b[1], w97665; CHECK-GI-NEXT:    ldr w9, [sp, #480]7666; CHECK-GI-NEXT:    movi d0, #00000000000000007667; CHECK-GI-NEXT:    mov v6.b[1], w87668; CHECK-GI-NEXT:    ldr w8, [sp, #96]7669; CHECK-GI-NEXT:    mov v4.b[2], w107670; CHECK-GI-NEXT:    ldr w10, [sp, #608]7671; CHECK-GI-NEXT:    mov v1.s[1], wzr7672; CHECK-GI-NEXT:    movi v22.2d, #00000000000000007673; CHECK-GI-NEXT:    mov v3.b[2], w87674; CHECK-GI-NEXT:    ldr w8, [sp, #352]7675; CHECK-GI-NEXT:    mov v2.b[3], w37676; CHECK-GI-NEXT:    mov v7.b[2], w107677; CHECK-GI-NEXT:    ldr w10, [sp, #488]7678; CHECK-GI-NEXT:    mov v0.s[1], wzr7679; CHECK-GI-NEXT:    mov v5.b[2], w87680; CHECK-GI-NEXT:    mov v6.b[2], w97681; CHECK-GI-NEXT:    ldr w8, [sp, #104]7682; CHECK-GI-NEXT:    ldr w9, [sp, #232]7683; CHECK-GI-NEXT:    movi v23.2d, #00000000000000007684; CHECK-GI-NEXT:    movi v24.2d, #00000000000000007685; CHECK-GI-NEXT:    mov v3.b[3], w87686; CHECK-GI-NEXT:    ldr w8, [sp, #360]7687; CHECK-GI-NEXT:    mov v2.b[4], w47688; CHECK-GI-NEXT:    mov v4.b[3], w97689; CHECK-GI-NEXT:    ldr w9, [sp, #616]7690; CHECK-GI-NEXT:    mov v1.s[2], wzr7691; CHECK-GI-NEXT:    mov v5.b[3], w87692; CHECK-GI-NEXT:    mov v6.b[3], w107693; CHECK-GI-NEXT:    ldr w8, [sp, #112]7694; CHECK-GI-NEXT:    ldr w10, [sp, #240]7695; CHECK-GI-NEXT:    mov v7.b[3], w97696; CHECK-GI-NEXT:    ldr w9, [sp, #496]7697; CHECK-GI-NEXT:    mov v3.b[4], w87698; CHECK-GI-NEXT:    ldr w8, [sp, #368]7699; CHECK-GI-NEXT:    mov v2.b[5], w57700; CHECK-GI-NEXT:    mov v4.b[4], w107701; CHECK-GI-NEXT:    ldr w10, [sp, #624]7702; CHECK-GI-NEXT:    movi v25.2d, #00000000000000007703; CHECK-GI-NEXT:    mov v5.b[4], w87704; CHECK-GI-NEXT:    mov v6.b[4], w97705; CHECK-GI-NEXT:    ldr w8, [sp, #120]7706; CHECK-GI-NEXT:    ldr w9, [sp, #248]7707; CHECK-GI-NEXT:    mov v7.b[4], w107708; CHECK-GI-NEXT:    ldr w10, [sp, #504]7709; CHECK-GI-NEXT:    mov v3.b[5], w87710; CHECK-GI-NEXT:    ldr w8, [sp, #376]7711; CHECK-GI-NEXT:    mov v2.b[6], w67712; CHECK-GI-NEXT:    mov v4.b[5], w97713; CHECK-GI-NEXT:    ldr w9, [sp, #632]7714; CHECK-GI-NEXT:    movi v26.2d, #00000000000000007715; CHECK-GI-NEXT:    mov v5.b[5], w87716; CHECK-GI-NEXT:    mov v6.b[5], w107717; CHECK-GI-NEXT:    ldr w8, [sp, #128]7718; CHECK-GI-NEXT:    ldr w10, [sp, #256]7719; CHECK-GI-NEXT:    mov v7.b[5], w97720; CHECK-GI-NEXT:    ldr w9, [sp, #512]7721; CHECK-GI-NEXT:    mov v3.b[6], w87722; CHECK-GI-NEXT:    ldr w8, [sp, #384]7723; CHECK-GI-NEXT:    mov v2.b[7], w77724; CHECK-GI-NEXT:    mov v4.b[6], w107725; CHECK-GI-NEXT:    ldr w10, [sp, #640]7726; CHECK-GI-NEXT:    movi v27.2d, #00000000000000007727; CHECK-GI-NEXT:    mov v5.b[6], w87728; CHECK-GI-NEXT:    ldr w8, [sp, #136]7729; CHECK-GI-NEXT:    mov v6.b[6], w97730; CHECK-GI-NEXT:    ldr w9, [sp, #264]7731; CHECK-GI-NEXT:    mov v7.b[6], w107732; CHECK-GI-NEXT:    ldr w10, [sp, #520]7733; CHECK-GI-NEXT:    mov v3.b[7], w87734; CHECK-GI-NEXT:    ldr w8, [sp, #392]7735; CHECK-GI-NEXT:    mov v2.b[8], w117736; CHECK-GI-NEXT:    mov v4.b[7], w97737; CHECK-GI-NEXT:    ldr w9, [sp, #648]7738; CHECK-GI-NEXT:    ldr w11, [sp, #24]7739; CHECK-GI-NEXT:    mov v5.b[7], w87740; CHECK-GI-NEXT:    ldr w8, [sp, #144]7741; CHECK-GI-NEXT:    mov v6.b[7], w107742; CHECK-GI-NEXT:    ldr w10, [sp, #272]7743; CHECK-GI-NEXT:    mov v7.b[7], w97744; CHECK-GI-NEXT:    ldr w9, [sp, #528]7745; CHECK-GI-NEXT:    mov v3.b[8], w87746; CHECK-GI-NEXT:    ldr w8, [sp, #400]7747; CHECK-GI-NEXT:    mov v2.b[9], w117748; CHECK-GI-NEXT:    mov v4.b[8], w107749; CHECK-GI-NEXT:    ldr w10, [sp, #656]7750; CHECK-GI-NEXT:    ldr w11, [sp, #32]7751; CHECK-GI-NEXT:    mov v5.b[8], w87752; CHECK-GI-NEXT:    ldr w8, [sp, #152]7753; CHECK-GI-NEXT:    mov v6.b[8], w97754; CHECK-GI-NEXT:    ldr w9, [sp, #280]7755; CHECK-GI-NEXT:    mov v7.b[8], w107756; CHECK-GI-NEXT:    ldr w10, [sp, #536]7757; CHECK-GI-NEXT:    mov v3.b[9], w87758; CHECK-GI-NEXT:    ldr w8, [sp, #408]7759; CHECK-GI-NEXT:    mov v2.b[10], w117760; CHECK-GI-NEXT:    mov v4.b[9], w97761; CHECK-GI-NEXT:    ldr w9, [sp, #664]7762; CHECK-GI-NEXT:    ldr w11, [sp, #40]7763; CHECK-GI-NEXT:    mov v5.b[9], w87764; CHECK-GI-NEXT:    ldr w8, [sp, #160]7765; CHECK-GI-NEXT:    mov v6.b[9], w107766; CHECK-GI-NEXT:    ldr w10, [sp, #288]7767; CHECK-GI-NEXT:    mov v7.b[9], w97768; CHECK-GI-NEXT:    ldr w9, [sp, #544]7769; CHECK-GI-NEXT:    mov v3.b[10], w87770; CHECK-GI-NEXT:    ldr w8, [sp, #416]7771; CHECK-GI-NEXT:    mov v2.b[11], w117772; CHECK-GI-NEXT:    mov v4.b[10], w107773; CHECK-GI-NEXT:    ldr w10, [sp, #672]7774; CHECK-GI-NEXT:    ldr w11, [sp, #48]7775; CHECK-GI-NEXT:    mov v5.b[10], w87776; CHECK-GI-NEXT:    ldr w8, [sp, #168]7777; CHECK-GI-NEXT:    mov v6.b[10], w97778; CHECK-GI-NEXT:    ldr w9, [sp, #296]7779; CHECK-GI-NEXT:    mov v7.b[10], w107780; CHECK-GI-NEXT:    ldr w10, [sp, #552]7781; CHECK-GI-NEXT:    mov v3.b[11], w87782; CHECK-GI-NEXT:    ldr w8, [sp, #424]7783; CHECK-GI-NEXT:    mov v2.b[12], w117784; CHECK-GI-NEXT:    mov v4.b[11], w97785; CHECK-GI-NEXT:    ldr w9, [sp, #680]7786; CHECK-GI-NEXT:    ldr w11, [sp, #56]7787; CHECK-GI-NEXT:    mov v5.b[11], w87788; CHECK-GI-NEXT:    ldr w8, [sp, #176]7789; CHECK-GI-NEXT:    mov v6.b[11], w107790; CHECK-GI-NEXT:    ldr w10, [sp, #304]7791; CHECK-GI-NEXT:    mov v7.b[11], w97792; CHECK-GI-NEXT:    ldr w9, [sp, #560]7793; CHECK-GI-NEXT:    mov v3.b[12], w87794; CHECK-GI-NEXT:    ldr w8, [sp, #432]7795; CHECK-GI-NEXT:    mov v2.b[13], w117796; CHECK-GI-NEXT:    mov v4.b[12], w107797; CHECK-GI-NEXT:    ldr w10, [sp, #688]7798; CHECK-GI-NEXT:    ldr w11, [sp, #64]7799; CHECK-GI-NEXT:    mov v5.b[12], w87800; CHECK-GI-NEXT:    ldr w8, [sp, #184]7801; CHECK-GI-NEXT:    mov v6.b[12], w97802; CHECK-GI-NEXT:    ldr w9, [sp, #312]7803; CHECK-GI-NEXT:    mov v7.b[12], w107804; CHECK-GI-NEXT:    ldr w10, [sp, #568]7805; CHECK-GI-NEXT:    mov v3.b[13], w87806; CHECK-GI-NEXT:    ldr w8, [sp, #440]7807; CHECK-GI-NEXT:    mov v2.b[14], w117808; CHECK-GI-NEXT:    mov v4.b[13], w97809; CHECK-GI-NEXT:    ldr w9, [sp, #696]7810; CHECK-GI-NEXT:    ldr w11, [sp, #72]7811; CHECK-GI-NEXT:    mov v5.b[13], w87812; CHECK-GI-NEXT:    ldr w8, [sp, #192]7813; CHECK-GI-NEXT:    mov v6.b[13], w107814; CHECK-GI-NEXT:    ldr w10, [sp, #320]7815; CHECK-GI-NEXT:    mov v7.b[13], w97816; CHECK-GI-NEXT:    ldr w9, [sp, #576]7817; CHECK-GI-NEXT:    mov v3.b[14], w87818; CHECK-GI-NEXT:    ldr w8, [sp, #448]7819; CHECK-GI-NEXT:    mov v2.b[15], w117820; CHECK-GI-NEXT:    mov v4.b[14], w107821; CHECK-GI-NEXT:    ldr w10, [sp, #704]7822; CHECK-GI-NEXT:    ldr w11, [sp, #848]7823; CHECK-GI-NEXT:    mov v5.b[14], w87824; CHECK-GI-NEXT:    ldr w8, [sp, #200]7825; CHECK-GI-NEXT:    mov v6.b[14], w97826; CHECK-GI-NEXT:    ldr w9, [sp, #328]7827; CHECK-GI-NEXT:    mov v7.b[14], w107828; CHECK-GI-NEXT:    ldr w10, [sp, #584]7829; CHECK-GI-NEXT:    mov v3.b[15], w87830; CHECK-GI-NEXT:    ldr w8, [sp, #456]7831; CHECK-GI-NEXT:    fmov s17, w117832; CHECK-GI-NEXT:    mov v4.b[15], w97833; CHECK-GI-NEXT:    ldr w9, [sp, #712]7834; CHECK-GI-NEXT:    ldr w11, [sp, #1104]7835; CHECK-GI-NEXT:    mov v5.b[15], w87836; CHECK-GI-NEXT:    ldr w8, [sp, #856]7837; CHECK-GI-NEXT:    mov v6.b[15], w107838; CHECK-GI-NEXT:    mov v7.b[15], w97839; CHECK-GI-NEXT:    ldr w9, [sp, #976]7840; CHECK-GI-NEXT:    ldr w10, [sp, #728]7841; CHECK-GI-NEXT:    mov v17.b[1], w87842; CHECK-GI-NEXT:    ldr w8, [sp, #1232]7843; CHECK-GI-NEXT:    fmov s19, w117844; CHECK-GI-NEXT:    fmov s18, w97845; CHECK-GI-NEXT:    ldr w9, [sp, #1360]7846; CHECK-GI-NEXT:    mov v16.b[1], w107847; CHECK-GI-NEXT:    ldr w10, [sp, #984]7848; CHECK-GI-NEXT:    fmov s20, w87849; CHECK-GI-NEXT:    ldr w11, [sp, #1240]7850; CHECK-GI-NEXT:    fmov s21, w97851; CHECK-GI-NEXT:    ldr w8, [sp, #1368]7852; CHECK-GI-NEXT:    mov v19.b[1], w127853; CHECK-GI-NEXT:    mov v18.b[1], w107854; CHECK-GI-NEXT:    ldr w10, [sp, #736]7855; CHECK-GI-NEXT:    ldr w9, [sp, #992]7856; CHECK-GI-NEXT:    mov v20.b[1], w117857; CHECK-GI-NEXT:    ldr w11, [sp, #864]7858; CHECK-GI-NEXT:    mov v0.s[2], wzr7859; CHECK-GI-NEXT:    mov v21.b[1], w87860; CHECK-GI-NEXT:    mov v16.b[2], w107861; CHECK-GI-NEXT:    ldr w8, [sp, #1120]7862; CHECK-GI-NEXT:    mov v17.b[2], w117863; CHECK-GI-NEXT:    ldr w10, [sp, #1248]7864; CHECK-GI-NEXT:    ldr w11, [sp, #744]7865; CHECK-GI-NEXT:    mov v18.b[2], w97866; CHECK-GI-NEXT:    ldr w9, [sp, #1376]7867; CHECK-GI-NEXT:    mov v19.b[2], w87868; CHECK-GI-NEXT:    ldr w8, [sp, #872]7869; CHECK-GI-NEXT:    mov v20.b[2], w107870; CHECK-GI-NEXT:    ldr w10, [sp, #1000]7871; CHECK-GI-NEXT:    mov v21.b[2], w97872; CHECK-GI-NEXT:    mov v16.b[3], w117873; CHECK-GI-NEXT:    ldr w9, [sp, #1256]7874; CHECK-GI-NEXT:    mov v17.b[3], w87875; CHECK-GI-NEXT:    ldr w8, [sp, #1128]7876; CHECK-GI-NEXT:    ldr w11, [sp, #752]7877; CHECK-GI-NEXT:    mov v18.b[3], w107878; CHECK-GI-NEXT:    ldr w10, [sp, #1384]7879; CHECK-GI-NEXT:    sdot v22.4s, v2.16b, v5.16b7880; CHECK-GI-NEXT:    mov v19.b[3], w87881; CHECK-GI-NEXT:    ldr w8, [sp, #880]7882; CHECK-GI-NEXT:    mov v20.b[3], w97883; CHECK-GI-NEXT:    ldr w9, [sp, #1008]7884; CHECK-GI-NEXT:    mov v21.b[3], w107885; CHECK-GI-NEXT:    mov v16.b[4], w117886; CHECK-GI-NEXT:    mov v17.b[4], w87887; CHECK-GI-NEXT:    ldr w8, [sp, #1136]7888; CHECK-GI-NEXT:    ldr w10, [sp, #1264]7889; CHECK-GI-NEXT:    mov v18.b[4], w97890; CHECK-GI-NEXT:    ldr w9, [sp, #1392]7891; CHECK-GI-NEXT:    ldr w11, [sp, #760]7892; CHECK-GI-NEXT:    mov v19.b[4], w87893; CHECK-GI-NEXT:    ldr w8, [sp, #888]7894; CHECK-GI-NEXT:    mov v20.b[4], w107895; CHECK-GI-NEXT:    ldr w10, [sp, #1016]7896; CHECK-GI-NEXT:    mov v21.b[4], w97897; CHECK-GI-NEXT:    mov v16.b[5], w117898; CHECK-GI-NEXT:    mov v17.b[5], w87899; CHECK-GI-NEXT:    ldr w8, [sp, #1144]7900; CHECK-GI-NEXT:    ldr w9, [sp, #1272]7901; CHECK-GI-NEXT:    mov v18.b[5], w107902; CHECK-GI-NEXT:    ldr w10, [sp, #1400]7903; CHECK-GI-NEXT:    ldr w11, [sp, #768]7904; CHECK-GI-NEXT:    mov v19.b[5], w87905; CHECK-GI-NEXT:    ldr w8, [sp, #896]7906; CHECK-GI-NEXT:    mov v20.b[5], w97907; CHECK-GI-NEXT:    ldr w9, [sp, #1024]7908; CHECK-GI-NEXT:    mov v21.b[5], w107909; CHECK-GI-NEXT:    mov v16.b[6], w117910; CHECK-GI-NEXT:    mov v17.b[6], w87911; CHECK-GI-NEXT:    ldr w8, [sp, #1152]7912; CHECK-GI-NEXT:    ldr w10, [sp, #1280]7913; CHECK-GI-NEXT:    mov v18.b[6], w97914; CHECK-GI-NEXT:    ldr w9, [sp, #1408]7915; CHECK-GI-NEXT:    ldr w11, [sp, #776]7916; CHECK-GI-NEXT:    mov v19.b[6], w87917; CHECK-GI-NEXT:    ldr w8, [sp, #904]7918; CHECK-GI-NEXT:    mov v20.b[6], w107919; CHECK-GI-NEXT:    ldr w10, [sp, #1032]7920; CHECK-GI-NEXT:    mov v21.b[6], w97921; CHECK-GI-NEXT:    mov v16.b[7], w117922; CHECK-GI-NEXT:    mov v17.b[7], w87923; CHECK-GI-NEXT:    ldr w8, [sp, #1160]7924; CHECK-GI-NEXT:    ldr w9, [sp, #1288]7925; CHECK-GI-NEXT:    mov v18.b[7], w107926; CHECK-GI-NEXT:    ldr w10, [sp, #1416]7927; CHECK-GI-NEXT:    ldr w11, [sp, #784]7928; CHECK-GI-NEXT:    mov v19.b[7], w87929; CHECK-GI-NEXT:    ldr w8, [sp, #912]7930; CHECK-GI-NEXT:    mov v20.b[7], w97931; CHECK-GI-NEXT:    ldr w9, [sp, #1040]7932; CHECK-GI-NEXT:    mov v21.b[7], w107933; CHECK-GI-NEXT:    mov v16.b[8], w117934; CHECK-GI-NEXT:    mov v17.b[8], w87935; CHECK-GI-NEXT:    ldr w8, [sp, #1168]7936; CHECK-GI-NEXT:    ldr w10, [sp, #1296]7937; CHECK-GI-NEXT:    mov v18.b[8], w97938; CHECK-GI-NEXT:    ldr w9, [sp, #1424]7939; CHECK-GI-NEXT:    ldr w11, [sp, #792]7940; CHECK-GI-NEXT:    mov v19.b[8], w87941; CHECK-GI-NEXT:    ldr w8, [sp, #920]7942; CHECK-GI-NEXT:    mov v20.b[8], w107943; CHECK-GI-NEXT:    ldr w10, [sp, #1048]7944; CHECK-GI-NEXT:    mov v21.b[8], w97945; CHECK-GI-NEXT:    mov v16.b[9], w117946; CHECK-GI-NEXT:    mov v17.b[9], w87947; CHECK-GI-NEXT:    ldr w8, [sp, #1176]7948; CHECK-GI-NEXT:    ldr w9, [sp, #1304]7949; CHECK-GI-NEXT:    mov v18.b[9], w107950; CHECK-GI-NEXT:    ldr w10, [sp, #1432]7951; CHECK-GI-NEXT:    ldr w11, [sp, #800]7952; CHECK-GI-NEXT:    mov v19.b[9], w87953; CHECK-GI-NEXT:    ldr w8, [sp, #928]7954; CHECK-GI-NEXT:    mov v20.b[9], w97955; CHECK-GI-NEXT:    ldr w9, [sp, #1056]7956; CHECK-GI-NEXT:    mov v21.b[9], w107957; CHECK-GI-NEXT:    mov v16.b[10], w117958; CHECK-GI-NEXT:    mov v17.b[10], w87959; CHECK-GI-NEXT:    ldr w8, [sp, #1184]7960; CHECK-GI-NEXT:    ldr w10, [sp, #1312]7961; CHECK-GI-NEXT:    mov v18.b[10], w97962; CHECK-GI-NEXT:    ldr w9, [sp, #1440]7963; CHECK-GI-NEXT:    ldr w11, [sp, #808]7964; CHECK-GI-NEXT:    mov v19.b[10], w87965; CHECK-GI-NEXT:    ldr w8, [sp, #936]7966; CHECK-GI-NEXT:    mov v20.b[10], w107967; CHECK-GI-NEXT:    ldr w10, [sp, #1064]7968; CHECK-GI-NEXT:    mov v21.b[10], w97969; CHECK-GI-NEXT:    mov v16.b[11], w117970; CHECK-GI-NEXT:    mov v17.b[11], w87971; CHECK-GI-NEXT:    ldr w8, [sp, #1192]7972; CHECK-GI-NEXT:    ldr w9, [sp, #1320]7973; CHECK-GI-NEXT:    mov v18.b[11], w107974; CHECK-GI-NEXT:    ldr w10, [sp, #1448]7975; CHECK-GI-NEXT:    ldr w11, [sp, #816]7976; CHECK-GI-NEXT:    mov v19.b[11], w87977; CHECK-GI-NEXT:    ldr w8, [sp, #944]7978; CHECK-GI-NEXT:    mov v20.b[11], w97979; CHECK-GI-NEXT:    ldr w9, [sp, #1072]7980; CHECK-GI-NEXT:    mov v21.b[11], w107981; CHECK-GI-NEXT:    mov v16.b[12], w117982; CHECK-GI-NEXT:    mov v17.b[12], w87983; CHECK-GI-NEXT:    ldr w8, [sp, #1200]7984; CHECK-GI-NEXT:    ldr w10, [sp, #1328]7985; CHECK-GI-NEXT:    mov v18.b[12], w97986; CHECK-GI-NEXT:    ldr w9, [sp, #1456]7987; CHECK-GI-NEXT:    ldr w11, [sp, #824]7988; CHECK-GI-NEXT:    mov v19.b[12], w87989; CHECK-GI-NEXT:    ldr w8, [sp, #952]7990; CHECK-GI-NEXT:    mov v20.b[12], w107991; CHECK-GI-NEXT:    ldr w10, [sp, #1080]7992; CHECK-GI-NEXT:    mov v21.b[12], w97993; CHECK-GI-NEXT:    mov v16.b[13], w117994; CHECK-GI-NEXT:    mov v17.b[13], w87995; CHECK-GI-NEXT:    ldr w8, [sp, #1208]7996; CHECK-GI-NEXT:    ldr w9, [sp, #1336]7997; CHECK-GI-NEXT:    mov v18.b[13], w107998; CHECK-GI-NEXT:    ldr w10, [sp, #1464]7999; CHECK-GI-NEXT:    ldr w11, [sp, #832]8000; CHECK-GI-NEXT:    mov v19.b[13], w88001; CHECK-GI-NEXT:    ldr w8, [sp, #960]8002; CHECK-GI-NEXT:    mov v20.b[13], w98003; CHECK-GI-NEXT:    ldr w9, [sp, #1088]8004; CHECK-GI-NEXT:    mov v21.b[13], w108005; CHECK-GI-NEXT:    mov v16.b[14], w118006; CHECK-GI-NEXT:    mov v17.b[14], w88007; CHECK-GI-NEXT:    ldr w8, [sp, #1216]8008; CHECK-GI-NEXT:    ldr w10, [sp, #1344]8009; CHECK-GI-NEXT:    mov v18.b[14], w98010; CHECK-GI-NEXT:    ldr w9, [sp, #1472]8011; CHECK-GI-NEXT:    ldr w11, [sp, #840]8012; CHECK-GI-NEXT:    mov v19.b[14], w88013; CHECK-GI-NEXT:    ldr w8, [sp, #968]8014; CHECK-GI-NEXT:    mov v20.b[14], w108015; CHECK-GI-NEXT:    mov v21.b[14], w98016; CHECK-GI-NEXT:    ldr w10, [sp, #1096]8017; CHECK-GI-NEXT:    mov v16.b[15], w118018; CHECK-GI-NEXT:    ldr w9, [sp, #1224]8019; CHECK-GI-NEXT:    ldr w11, [sp, #1352]8020; CHECK-GI-NEXT:    mov v17.b[15], w88021; CHECK-GI-NEXT:    ldr w8, [sp, #1480]8022; CHECK-GI-NEXT:    mov v18.b[15], w108023; CHECK-GI-NEXT:    sdot v23.4s, v3.16b, v6.16b8024; CHECK-GI-NEXT:    mov v19.b[15], w98025; CHECK-GI-NEXT:    mov v20.b[15], w118026; CHECK-GI-NEXT:    sdot v24.4s, v4.16b, v7.16b8027; CHECK-GI-NEXT:    mov v21.b[15], w88028; CHECK-GI-NEXT:    mov v1.s[3], wzr8029; CHECK-GI-NEXT:    mov v0.s[3], wzr8030; CHECK-GI-NEXT:    add v2.4s, v22.4s, v23.4s8031; CHECK-GI-NEXT:    sdot v25.4s, v16.16b, v19.16b8032; CHECK-GI-NEXT:    sdot v27.4s, v17.16b, v20.16b8033; CHECK-GI-NEXT:    sdot v26.4s, v18.16b, v21.16b8034; CHECK-GI-NEXT:    add v1.4s, v24.4s, v1.4s8035; CHECK-GI-NEXT:    add v3.4s, v25.4s, v27.4s8036; CHECK-GI-NEXT:    add v1.4s, v2.4s, v1.4s8037; CHECK-GI-NEXT:    add v0.4s, v26.4s, v0.4s8038; CHECK-GI-NEXT:    addv s1, v1.4s8039; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s8040; CHECK-GI-NEXT:    fmov w8, s18041; CHECK-GI-NEXT:    addv s0, v0.4s8042; CHECK-GI-NEXT:    fmov w9, s08043; CHECK-GI-NEXT:    add w0, w8, w98044; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload8045; CHECK-GI-NEXT:    ret8046entry:8047  %az = sext <48 x i8> %a to <48 x i32>8048  %bz = sext <48 x i8> %b to <48 x i32>8049  %m1 = mul nuw nsw <48 x i32> %az, %bz8050  %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m1)8051  %cz = sext <48 x i8> %c to <48 x i32>8052  %dz = sext <48 x i8> %d to <48 x i32>8053  %m2 = mul nuw nsw <48 x i32> %cz, %dz8054  %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m2)8055  %x = add i32 %r1, %r28056  ret i32 %x8057}8058 8059define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {8060; CHECK-SD-LABEL: test_sdot_v48i8_double_nomla:8061; CHECK-SD:       // %bb.0: // %entry8062; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill8063; CHECK-SD-NEXT:    .cfi_def_cfa_offset 168064; CHECK-SD-NEXT:    .cfi_offset w29, -168065; CHECK-SD-NEXT:    ldr b0, [sp, #208]8066; CHECK-SD-NEXT:    add x8, sp, #2168067; CHECK-SD-NEXT:    add x9, sp, #2728068; CHECK-SD-NEXT:    ldr b2, [sp, #80]8069; CHECK-SD-NEXT:    ldr b4, [sp, #976]8070; CHECK-SD-NEXT:    ldr b6, [sp, #720]8071; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]8072; CHECK-SD-NEXT:    add x8, sp, #2248073; CHECK-SD-NEXT:    fmov s16, w08074; CHECK-SD-NEXT:    ldr b17, [sp, #848]8075; CHECK-SD-NEXT:    add x10, sp, #248076; CHECK-SD-NEXT:    movi v19.2d, #00000000000000008077; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]8078; CHECK-SD-NEXT:    add x8, sp, #2328079; CHECK-SD-NEXT:    mov v16.b[1], w18080; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x8]8081; CHECK-SD-NEXT:    add x8, sp, #2408082; CHECK-SD-NEXT:    mov v16.b[2], w28083; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x8]8084; CHECK-SD-NEXT:    add x8, sp, #2488085; CHECK-SD-NEXT:    mov v16.b[3], w38086; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x8]8087; CHECK-SD-NEXT:    add x8, sp, #2568088; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x8]8089; CHECK-SD-NEXT:    add x8, sp, #2648090; CHECK-SD-NEXT:    mov v16.b[4], w48091; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x8]8092; CHECK-SD-NEXT:    ldr b1, [x9]8093; CHECK-SD-NEXT:    add x8, sp, #2808094; CHECK-SD-NEXT:    add x9, sp, #888095; CHECK-SD-NEXT:    mov v16.b[5], w58096; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x8]8097; CHECK-SD-NEXT:    add x8, sp, #2888098; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x8]8099; CHECK-SD-NEXT:    add x8, sp, #2968100; CHECK-SD-NEXT:    mov v16.b[6], w68101; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]8102; CHECK-SD-NEXT:    add x8, sp, #3048103; CHECK-SD-NEXT:    mov v16.b[7], w78104; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x8]8105; CHECK-SD-NEXT:    add x8, sp, #3128106; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x8]8107; CHECK-SD-NEXT:    add x8, sp, #3208108; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x8]8109; CHECK-SD-NEXT:    add x8, sp, #3288110; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x8]8111; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x9]8112; CHECK-SD-NEXT:    add x8, sp, #968113; CHECK-SD-NEXT:    add x9, sp, #1448114; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x8]8115; CHECK-SD-NEXT:    add x8, sp, #1048116; CHECK-SD-NEXT:    zip1 v0.2d, v0.2d, v1.2d8117; CHECK-SD-NEXT:    movi v1.16b, #18118; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x8]8119; CHECK-SD-NEXT:    add x8, sp, #1128120; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x8]8121; CHECK-SD-NEXT:    add x8, sp, #1208122; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x8]8123; CHECK-SD-NEXT:    add x8, sp, #1288124; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x8]8125; CHECK-SD-NEXT:    add x8, sp, #1368126; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x8]8127; CHECK-SD-NEXT:    ldr b3, [x9]8128; CHECK-SD-NEXT:    add x8, sp, #1528129; CHECK-SD-NEXT:    add x9, sp, #9848130; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x8]8131; CHECK-SD-NEXT:    add x8, sp, #1608132; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x8]8133; CHECK-SD-NEXT:    add x8, sp, #1688134; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x8]8135; CHECK-SD-NEXT:    add x8, sp, #1768136; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x8]8137; CHECK-SD-NEXT:    add x8, sp, #1848138; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x8]8139; CHECK-SD-NEXT:    add x8, sp, #1928140; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x8]8141; CHECK-SD-NEXT:    add x8, sp, #2008142; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x8]8143; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x9]8144; CHECK-SD-NEXT:    add x8, sp, #9928145; CHECK-SD-NEXT:    add x9, sp, #10408146; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x8]8147; CHECK-SD-NEXT:    add x8, sp, #10008148; CHECK-SD-NEXT:    zip1 v2.2d, v2.2d, v3.2d8149; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x8]8150; CHECK-SD-NEXT:    add x8, sp, #10088151; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]8152; CHECK-SD-NEXT:    add x8, sp, #10168153; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x8]8154; CHECK-SD-NEXT:    add x8, sp, #10248155; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x8]8156; CHECK-SD-NEXT:    add x8, sp, #10328157; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x8]8158; CHECK-SD-NEXT:    ldr b5, [x9]8159; CHECK-SD-NEXT:    add x8, sp, #10488160; CHECK-SD-NEXT:    add x9, sp, #7288161; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x8]8162; CHECK-SD-NEXT:    add x8, sp, #10568163; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x8]8164; CHECK-SD-NEXT:    add x8, sp, #10648165; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x8]8166; CHECK-SD-NEXT:    add x8, sp, #10728167; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x8]8168; CHECK-SD-NEXT:    add x8, sp, #10808169; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x8]8170; CHECK-SD-NEXT:    add x8, sp, #10888171; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x8]8172; CHECK-SD-NEXT:    add x8, sp, #10968173; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x8]8174; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x9]8175; CHECK-SD-NEXT:    add x8, sp, #7368176; CHECK-SD-NEXT:    add x9, sp, #7848177; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]8178; CHECK-SD-NEXT:    add x8, sp, #7448179; CHECK-SD-NEXT:    zip1 v4.2d, v4.2d, v5.2d8180; CHECK-SD-NEXT:    movi v5.2d, #00000000000000008181; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x8]8182; CHECK-SD-NEXT:    add x8, sp, #7528183; CHECK-SD-NEXT:    sdot v19.4s, v4.16b, v1.16b8184; CHECK-SD-NEXT:    sdot v5.4s, v0.16b, v1.16b8185; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x8]8186; CHECK-SD-NEXT:    add x8, sp, #7608187; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x8]8188; CHECK-SD-NEXT:    add x8, sp, #7688189; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x8]8190; CHECK-SD-NEXT:    add x8, sp, #7768191; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x8]8192; CHECK-SD-NEXT:    ldr b7, [x9]8193; CHECK-SD-NEXT:    add x8, sp, #7928194; CHECK-SD-NEXT:    add x9, sp, #8568195; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x8]8196; CHECK-SD-NEXT:    add x8, sp, #8008197; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x8]8198; CHECK-SD-NEXT:    add x8, sp, #8088199; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x8]8200; CHECK-SD-NEXT:    add x8, sp, #8168201; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x8]8202; CHECK-SD-NEXT:    add x8, sp, #8248203; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x8]8204; CHECK-SD-NEXT:    add x8, sp, #8328205; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x8]8206; CHECK-SD-NEXT:    add x8, sp, #8408207; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x8]8208; CHECK-SD-NEXT:    ld1 { v17.b }[1], [x9]8209; CHECK-SD-NEXT:    add x8, sp, #8648210; CHECK-SD-NEXT:    add x9, sp, #168211; CHECK-SD-NEXT:    ld1 { v16.b }[8], [x9]8212; CHECK-SD-NEXT:    add x9, sp, #9128213; CHECK-SD-NEXT:    ld1 { v17.b }[2], [x8]8214; CHECK-SD-NEXT:    add x8, sp, #8728215; CHECK-SD-NEXT:    zip1 v0.2d, v6.2d, v7.2d8216; CHECK-SD-NEXT:    ld1 { v16.b }[9], [x10]8217; CHECK-SD-NEXT:    ld1 { v17.b }[3], [x8]8218; CHECK-SD-NEXT:    add x8, sp, #8808219; CHECK-SD-NEXT:    sdot v19.4s, v0.16b, v1.16b8220; CHECK-SD-NEXT:    ld1 { v17.b }[4], [x8]8221; CHECK-SD-NEXT:    add x8, sp, #8888222; CHECK-SD-NEXT:    ld1 { v17.b }[5], [x8]8223; CHECK-SD-NEXT:    add x8, sp, #8968224; CHECK-SD-NEXT:    ld1 { v17.b }[6], [x8]8225; CHECK-SD-NEXT:    add x8, sp, #9048226; CHECK-SD-NEXT:    ld1 { v17.b }[7], [x8]8227; CHECK-SD-NEXT:    ldr b18, [x9]8228; CHECK-SD-NEXT:    add x8, sp, #9208229; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x8]8230; CHECK-SD-NEXT:    add x8, sp, #328231; CHECK-SD-NEXT:    ld1 { v16.b }[10], [x8]8232; CHECK-SD-NEXT:    add x8, sp, #9288233; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x8]8234; CHECK-SD-NEXT:    add x8, sp, #408235; CHECK-SD-NEXT:    ld1 { v16.b }[11], [x8]8236; CHECK-SD-NEXT:    add x8, sp, #9368237; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x8]8238; CHECK-SD-NEXT:    add x8, sp, #488239; CHECK-SD-NEXT:    ld1 { v16.b }[12], [x8]8240; CHECK-SD-NEXT:    add x8, sp, #9448241; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x8]8242; CHECK-SD-NEXT:    add x8, sp, #568243; CHECK-SD-NEXT:    ld1 { v16.b }[13], [x8]8244; CHECK-SD-NEXT:    add x8, sp, #9528245; CHECK-SD-NEXT:    ld1 { v18.b }[5], [x8]8246; CHECK-SD-NEXT:    add x8, sp, #648247; CHECK-SD-NEXT:    ld1 { v16.b }[14], [x8]8248; CHECK-SD-NEXT:    add x8, sp, #9608249; CHECK-SD-NEXT:    ld1 { v18.b }[6], [x8]8250; CHECK-SD-NEXT:    add x8, sp, #728251; CHECK-SD-NEXT:    ld1 { v16.b }[15], [x8]8252; CHECK-SD-NEXT:    add x8, sp, #9688253; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x8]8254; CHECK-SD-NEXT:    sdot v5.4s, v16.16b, v1.16b8255; CHECK-SD-NEXT:    zip1 v0.2d, v17.2d, v18.2d8256; CHECK-SD-NEXT:    sdot v5.4s, v2.16b, v1.16b8257; CHECK-SD-NEXT:    sdot v19.4s, v0.16b, v1.16b8258; CHECK-SD-NEXT:    add v0.4s, v5.4s, v19.4s8259; CHECK-SD-NEXT:    addv s0, v0.4s8260; CHECK-SD-NEXT:    fmov w0, s08261; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload8262; CHECK-SD-NEXT:    ret8263;8264; CHECK-GI-LABEL: test_sdot_v48i8_double_nomla:8265; CHECK-GI:       // %bb.0: // %entry8266; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill8267; CHECK-GI-NEXT:    .cfi_def_cfa_offset 168268; CHECK-GI-NEXT:    .cfi_offset w29, -168269; CHECK-GI-NEXT:    ldr w9, [sp, #208]8270; CHECK-GI-NEXT:    ldr w11, [sp, #80]8271; CHECK-GI-NEXT:    fmov s2, w08272; CHECK-GI-NEXT:    ldr w10, [sp, #216]8273; CHECK-GI-NEXT:    ldr w12, [sp, #848]8274; CHECK-GI-NEXT:    movi d1, #00000000000000008275; CHECK-GI-NEXT:    fmov s4, w98276; CHECK-GI-NEXT:    fmov s3, w118277; CHECK-GI-NEXT:    ldr w11, [sp, #720]8278; CHECK-GI-NEXT:    ldr w8, [sp, #88]8279; CHECK-GI-NEXT:    fmov s6, w128280; CHECK-GI-NEXT:    ldr w9, [sp, #856]8281; CHECK-GI-NEXT:    fmov s5, w118282; CHECK-GI-NEXT:    ldr w11, [sp, #984]8283; CHECK-GI-NEXT:    mov v2.b[1], w18284; CHECK-GI-NEXT:    mov v4.b[1], w108285; CHECK-GI-NEXT:    ldr w10, [sp, #976]8286; CHECK-GI-NEXT:    mov v3.b[1], w88287; CHECK-GI-NEXT:    ldr w8, [sp, #728]8288; CHECK-GI-NEXT:    mov v6.b[1], w98289; CHECK-GI-NEXT:    ldr w9, [sp, #736]8290; CHECK-GI-NEXT:    fmov s7, w108291; CHECK-GI-NEXT:    ldr w10, [sp, #96]8292; CHECK-GI-NEXT:    ldr w12, [sp, #16]8293; CHECK-GI-NEXT:    mov v5.b[1], w88294; CHECK-GI-NEXT:    ldr w8, [sp, #224]8295; CHECK-GI-NEXT:    mov v2.b[2], w28296; CHECK-GI-NEXT:    mov v3.b[2], w108297; CHECK-GI-NEXT:    ldr w10, [sp, #864]8298; CHECK-GI-NEXT:    movi d0, #00000000000000008299; CHECK-GI-NEXT:    mov v7.b[1], w118300; CHECK-GI-NEXT:    ldr w11, [sp, #992]8301; CHECK-GI-NEXT:    mov v4.b[2], w88302; CHECK-GI-NEXT:    ldr w8, [sp, #104]8303; CHECK-GI-NEXT:    mov v6.b[2], w108304; CHECK-GI-NEXT:    ldr w10, [sp, #872]8305; CHECK-GI-NEXT:    mov v5.b[2], w98306; CHECK-GI-NEXT:    ldr w9, [sp, #232]8307; CHECK-GI-NEXT:    mov v2.b[3], w38308; CHECK-GI-NEXT:    mov v3.b[3], w88309; CHECK-GI-NEXT:    ldr w8, [sp, #744]8310; CHECK-GI-NEXT:    mov v1.s[1], wzr8311; CHECK-GI-NEXT:    mov v7.b[2], w118312; CHECK-GI-NEXT:    ldr w11, [sp, #1000]8313; CHECK-GI-NEXT:    mov v4.b[3], w98314; CHECK-GI-NEXT:    ldr w9, [sp, #112]8315; CHECK-GI-NEXT:    mov v6.b[3], w108316; CHECK-GI-NEXT:    ldr w10, [sp, #880]8317; CHECK-GI-NEXT:    mov v5.b[3], w88318; CHECK-GI-NEXT:    ldr w8, [sp, #240]8319; CHECK-GI-NEXT:    mov v2.b[4], w48320; CHECK-GI-NEXT:    mov v3.b[4], w98321; CHECK-GI-NEXT:    ldr w9, [sp, #752]8322; CHECK-GI-NEXT:    mov v0.s[1], wzr8323; CHECK-GI-NEXT:    mov v7.b[3], w118324; CHECK-GI-NEXT:    ldr w11, [sp, #1008]8325; CHECK-GI-NEXT:    mov v4.b[4], w88326; CHECK-GI-NEXT:    ldr w8, [sp, #120]8327; CHECK-GI-NEXT:    mov v6.b[4], w108328; CHECK-GI-NEXT:    ldr w10, [sp, #888]8329; CHECK-GI-NEXT:    mov v5.b[4], w98330; CHECK-GI-NEXT:    ldr w9, [sp, #248]8331; CHECK-GI-NEXT:    mov v2.b[5], w58332; CHECK-GI-NEXT:    mov v3.b[5], w88333; CHECK-GI-NEXT:    ldr w8, [sp, #760]8334; CHECK-GI-NEXT:    movi v16.16b, #18335; CHECK-GI-NEXT:    mov v7.b[4], w118336; CHECK-GI-NEXT:    ldr w11, [sp, #1016]8337; CHECK-GI-NEXT:    mov v4.b[5], w98338; CHECK-GI-NEXT:    ldr w9, [sp, #128]8339; CHECK-GI-NEXT:    mov v6.b[5], w108340; CHECK-GI-NEXT:    ldr w10, [sp, #896]8341; CHECK-GI-NEXT:    mov v5.b[5], w88342; CHECK-GI-NEXT:    ldr w8, [sp, #256]8343; CHECK-GI-NEXT:    mov v2.b[6], w68344; CHECK-GI-NEXT:    mov v3.b[6], w98345; CHECK-GI-NEXT:    ldr w9, [sp, #768]8346; CHECK-GI-NEXT:    movi v17.2d, #00000000000000008347; CHECK-GI-NEXT:    mov v7.b[5], w118348; CHECK-GI-NEXT:    ldr w11, [sp, #1024]8349; CHECK-GI-NEXT:    mov v4.b[6], w88350; CHECK-GI-NEXT:    ldr w8, [sp, #136]8351; CHECK-GI-NEXT:    mov v6.b[6], w108352; CHECK-GI-NEXT:    ldr w10, [sp, #904]8353; CHECK-GI-NEXT:    mov v5.b[6], w98354; CHECK-GI-NEXT:    ldr w9, [sp, #264]8355; CHECK-GI-NEXT:    mov v2.b[7], w78356; CHECK-GI-NEXT:    mov v3.b[7], w88357; CHECK-GI-NEXT:    ldr w8, [sp, #776]8358; CHECK-GI-NEXT:    movi v18.2d, #00000000000000008359; CHECK-GI-NEXT:    mov v7.b[6], w118360; CHECK-GI-NEXT:    ldr w11, [sp, #1032]8361; CHECK-GI-NEXT:    mov v4.b[7], w98362; CHECK-GI-NEXT:    ldr w9, [sp, #144]8363; CHECK-GI-NEXT:    mov v6.b[7], w108364; CHECK-GI-NEXT:    ldr w10, [sp, #912]8365; CHECK-GI-NEXT:    mov v5.b[7], w88366; CHECK-GI-NEXT:    ldr w8, [sp, #272]8367; CHECK-GI-NEXT:    mov v2.b[8], w128368; CHECK-GI-NEXT:    mov v3.b[8], w98369; CHECK-GI-NEXT:    ldr w9, [sp, #784]8370; CHECK-GI-NEXT:    ldr w12, [sp, #24]8371; CHECK-GI-NEXT:    mov v7.b[7], w118372; CHECK-GI-NEXT:    ldr w11, [sp, #1040]8373; CHECK-GI-NEXT:    mov v4.b[8], w88374; CHECK-GI-NEXT:    ldr w8, [sp, #152]8375; CHECK-GI-NEXT:    mov v6.b[8], w108376; CHECK-GI-NEXT:    ldr w10, [sp, #920]8377; CHECK-GI-NEXT:    mov v5.b[8], w98378; CHECK-GI-NEXT:    ldr w9, [sp, #280]8379; CHECK-GI-NEXT:    mov v2.b[9], w128380; CHECK-GI-NEXT:    mov v3.b[9], w88381; CHECK-GI-NEXT:    ldr w8, [sp, #792]8382; CHECK-GI-NEXT:    ldr w12, [sp, #32]8383; CHECK-GI-NEXT:    mov v7.b[8], w118384; CHECK-GI-NEXT:    ldr w11, [sp, #1048]8385; CHECK-GI-NEXT:    mov v4.b[9], w98386; CHECK-GI-NEXT:    ldr w9, [sp, #160]8387; CHECK-GI-NEXT:    mov v6.b[9], w108388; CHECK-GI-NEXT:    ldr w10, [sp, #928]8389; CHECK-GI-NEXT:    mov v5.b[9], w88390; CHECK-GI-NEXT:    ldr w8, [sp, #288]8391; CHECK-GI-NEXT:    mov v2.b[10], w128392; CHECK-GI-NEXT:    mov v3.b[10], w98393; CHECK-GI-NEXT:    ldr w9, [sp, #800]8394; CHECK-GI-NEXT:    ldr w12, [sp, #40]8395; CHECK-GI-NEXT:    mov v7.b[9], w118396; CHECK-GI-NEXT:    ldr w11, [sp, #1056]8397; CHECK-GI-NEXT:    mov v4.b[10], w88398; CHECK-GI-NEXT:    ldr w8, [sp, #168]8399; CHECK-GI-NEXT:    mov v6.b[10], w108400; CHECK-GI-NEXT:    ldr w10, [sp, #936]8401; CHECK-GI-NEXT:    mov v5.b[10], w98402; CHECK-GI-NEXT:    ldr w9, [sp, #296]8403; CHECK-GI-NEXT:    mov v2.b[11], w128404; CHECK-GI-NEXT:    mov v3.b[11], w88405; CHECK-GI-NEXT:    ldr w8, [sp, #808]8406; CHECK-GI-NEXT:    ldr w12, [sp, #48]8407; CHECK-GI-NEXT:    mov v7.b[10], w118408; CHECK-GI-NEXT:    ldr w11, [sp, #1064]8409; CHECK-GI-NEXT:    mov v4.b[11], w98410; CHECK-GI-NEXT:    ldr w9, [sp, #176]8411; CHECK-GI-NEXT:    mov v6.b[11], w108412; CHECK-GI-NEXT:    ldr w10, [sp, #944]8413; CHECK-GI-NEXT:    mov v5.b[11], w88414; CHECK-GI-NEXT:    ldr w8, [sp, #304]8415; CHECK-GI-NEXT:    mov v2.b[12], w128416; CHECK-GI-NEXT:    mov v3.b[12], w98417; CHECK-GI-NEXT:    ldr w9, [sp, #816]8418; CHECK-GI-NEXT:    ldr w12, [sp, #56]8419; CHECK-GI-NEXT:    mov v7.b[11], w118420; CHECK-GI-NEXT:    ldr w11, [sp, #1072]8421; CHECK-GI-NEXT:    mov v4.b[12], w88422; CHECK-GI-NEXT:    ldr w8, [sp, #184]8423; CHECK-GI-NEXT:    mov v6.b[12], w108424; CHECK-GI-NEXT:    ldr w10, [sp, #952]8425; CHECK-GI-NEXT:    mov v5.b[12], w98426; CHECK-GI-NEXT:    ldr w9, [sp, #312]8427; CHECK-GI-NEXT:    mov v2.b[13], w128428; CHECK-GI-NEXT:    mov v3.b[13], w88429; CHECK-GI-NEXT:    ldr w8, [sp, #824]8430; CHECK-GI-NEXT:    ldr w12, [sp, #64]8431; CHECK-GI-NEXT:    mov v7.b[12], w118432; CHECK-GI-NEXT:    ldr w11, [sp, #1080]8433; CHECK-GI-NEXT:    mov v4.b[13], w98434; CHECK-GI-NEXT:    ldr w9, [sp, #192]8435; CHECK-GI-NEXT:    mov v6.b[13], w108436; CHECK-GI-NEXT:    ldr w10, [sp, #960]8437; CHECK-GI-NEXT:    mov v5.b[13], w88438; CHECK-GI-NEXT:    ldr w8, [sp, #320]8439; CHECK-GI-NEXT:    mov v2.b[14], w128440; CHECK-GI-NEXT:    mov v3.b[14], w98441; CHECK-GI-NEXT:    ldr w9, [sp, #832]8442; CHECK-GI-NEXT:    ldr w12, [sp, #72]8443; CHECK-GI-NEXT:    mov v7.b[13], w118444; CHECK-GI-NEXT:    ldr w11, [sp, #1088]8445; CHECK-GI-NEXT:    mov v4.b[14], w88446; CHECK-GI-NEXT:    ldr w8, [sp, #200]8447; CHECK-GI-NEXT:    mov v6.b[14], w108448; CHECK-GI-NEXT:    ldr w10, [sp, #968]8449; CHECK-GI-NEXT:    mov v5.b[14], w98450; CHECK-GI-NEXT:    ldr w9, [sp, #328]8451; CHECK-GI-NEXT:    mov v2.b[15], w128452; CHECK-GI-NEXT:    mov v3.b[15], w88453; CHECK-GI-NEXT:    ldr w8, [sp, #840]8454; CHECK-GI-NEXT:    movi v19.2d, #00000000000000008455; CHECK-GI-NEXT:    mov v7.b[14], w118456; CHECK-GI-NEXT:    ldr w11, [sp, #1096]8457; CHECK-GI-NEXT:    mov v4.b[15], w98458; CHECK-GI-NEXT:    mov v6.b[15], w108459; CHECK-GI-NEXT:    movi v20.2d, #00000000000000008460; CHECK-GI-NEXT:    movi v21.2d, #00000000000000008461; CHECK-GI-NEXT:    mov v5.b[15], w88462; CHECK-GI-NEXT:    movi v22.2d, #00000000000000008463; CHECK-GI-NEXT:    mov v1.s[2], wzr8464; CHECK-GI-NEXT:    mov v0.s[2], wzr8465; CHECK-GI-NEXT:    sdot v17.4s, v2.16b, v16.16b8466; CHECK-GI-NEXT:    sdot v18.4s, v3.16b, v16.16b8467; CHECK-GI-NEXT:    mov v7.b[15], w118468; CHECK-GI-NEXT:    sdot v19.4s, v4.16b, v16.16b8469; CHECK-GI-NEXT:    sdot v21.4s, v6.16b, v16.16b8470; CHECK-GI-NEXT:    sdot v20.4s, v5.16b, v16.16b8471; CHECK-GI-NEXT:    mov v1.s[3], wzr8472; CHECK-GI-NEXT:    mov v0.s[3], wzr8473; CHECK-GI-NEXT:    add v2.4s, v17.4s, v18.4s8474; CHECK-GI-NEXT:    sdot v22.4s, v7.16b, v16.16b8475; CHECK-GI-NEXT:    add v1.4s, v19.4s, v1.4s8476; CHECK-GI-NEXT:    add v3.4s, v20.4s, v21.4s8477; CHECK-GI-NEXT:    add v0.4s, v22.4s, v0.4s8478; CHECK-GI-NEXT:    add v1.4s, v2.4s, v1.4s8479; CHECK-GI-NEXT:    add v0.4s, v3.4s, v0.4s8480; CHECK-GI-NEXT:    addv s1, v1.4s8481; CHECK-GI-NEXT:    addv s0, v0.4s8482; CHECK-GI-NEXT:    fmov w8, s18483; CHECK-GI-NEXT:    fmov w9, s08484; CHECK-GI-NEXT:    add w0, w8, w98485; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload8486; CHECK-GI-NEXT:    ret8487entry:8488  %az = sext <48 x i8> %a to <48 x i32>8489  %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %az)8490  %cz = sext <48 x i8> %c to <48 x i32>8491  %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %cz)8492  %x = add i32 %r1, %r28493  ret i32 %x8494}8495 8496define i32 @test_udot_v64i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {8497; CHECK-SD-LABEL: test_udot_v64i8:8498; CHECK-SD:       // %bb.0: // %entry8499; CHECK-SD-NEXT:    movi v0.2d, #00000000000000008500; CHECK-SD-NEXT:    movi v1.2d, #00000000000000008501; CHECK-SD-NEXT:    ldp q2, q3, [x0, #32]8502; CHECK-SD-NEXT:    ldp q4, q5, [x1, #32]8503; CHECK-SD-NEXT:    udot v1.4s, v5.16b, v3.16b8504; CHECK-SD-NEXT:    udot v0.4s, v4.16b, v2.16b8505; CHECK-SD-NEXT:    ldp q2, q3, [x0]8506; CHECK-SD-NEXT:    ldp q4, q5, [x1]8507; CHECK-SD-NEXT:    udot v1.4s, v5.16b, v3.16b8508; CHECK-SD-NEXT:    udot v0.4s, v4.16b, v2.16b8509; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s8510; CHECK-SD-NEXT:    addv s0, v0.4s8511; CHECK-SD-NEXT:    fmov w8, s08512; CHECK-SD-NEXT:    add w0, w8, w28513; CHECK-SD-NEXT:    ret8514;8515; CHECK-GI-LABEL: test_udot_v64i8:8516; CHECK-GI:       // %bb.0: // %entry8517; CHECK-GI-NEXT:    movi v0.2d, #00000000000000008518; CHECK-GI-NEXT:    movi v3.2d, #00000000000000008519; CHECK-GI-NEXT:    movi v4.2d, #00000000000000008520; CHECK-GI-NEXT:    movi v5.2d, #00000000000000008521; CHECK-GI-NEXT:    ldp q1, q2, [x0]8522; CHECK-GI-NEXT:    ldp q6, q7, [x0, #32]8523; CHECK-GI-NEXT:    ldp q16, q17, [x1]8524; CHECK-GI-NEXT:    ldp q18, q19, [x1, #32]8525; CHECK-GI-NEXT:    udot v0.4s, v16.16b, v1.16b8526; CHECK-GI-NEXT:    udot v4.4s, v17.16b, v2.16b8527; CHECK-GI-NEXT:    udot v5.4s, v18.16b, v6.16b8528; CHECK-GI-NEXT:    udot v3.4s, v19.16b, v7.16b8529; CHECK-GI-NEXT:    add v0.4s, v0.4s, v4.4s8530; CHECK-GI-NEXT:    add v1.4s, v5.4s, v3.4s8531; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s8532; CHECK-GI-NEXT:    addv s0, v0.4s8533; CHECK-GI-NEXT:    fmov w8, s08534; CHECK-GI-NEXT:    add w0, w8, w28535; CHECK-GI-NEXT:    ret8536entry:8537  %0 = load <64 x i8>, ptr %a8538  %1 = zext <64 x i8> %0 to <64 x i32>8539  %2 = load <64 x i8>, ptr %b8540  %3 = zext <64 x i8> %2 to <64 x i32>8541  %4 = mul nuw nsw <64 x i32> %3, %18542  %5 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %4)8543  %op.extra = add i32 %5, %sum8544  ret i32 %op.extra8545}8546 8547define i32 @test_udot_v64i8_nomla(ptr nocapture readonly %a1) {8548; CHECK-SD-LABEL: test_udot_v64i8_nomla:8549; CHECK-SD:       // %bb.0: // %entry8550; CHECK-SD-NEXT:    movi v0.16b, #18551; CHECK-SD-NEXT:    movi v1.2d, #00000000000000008552; CHECK-SD-NEXT:    movi v2.2d, #00000000000000008553; CHECK-SD-NEXT:    ldp q3, q4, [x0, #32]8554; CHECK-SD-NEXT:    udot v2.4s, v4.16b, v0.16b8555; CHECK-SD-NEXT:    udot v1.4s, v3.16b, v0.16b8556; CHECK-SD-NEXT:    ldp q3, q4, [x0]8557; CHECK-SD-NEXT:    udot v2.4s, v4.16b, v0.16b8558; CHECK-SD-NEXT:    udot v1.4s, v3.16b, v0.16b8559; CHECK-SD-NEXT:    add v0.4s, v1.4s, v2.4s8560; CHECK-SD-NEXT:    addv s0, v0.4s8561; CHECK-SD-NEXT:    fmov w0, s08562; CHECK-SD-NEXT:    ret8563;8564; CHECK-GI-LABEL: test_udot_v64i8_nomla:8565; CHECK-GI:       // %bb.0: // %entry8566; CHECK-GI-NEXT:    movi v0.16b, #18567; CHECK-GI-NEXT:    movi v1.2d, #00000000000000008568; CHECK-GI-NEXT:    movi v2.2d, #00000000000000008569; CHECK-GI-NEXT:    movi v3.2d, #00000000000000008570; CHECK-GI-NEXT:    ldp q5, q6, [x0]8571; CHECK-GI-NEXT:    movi v4.2d, #00000000000000008572; CHECK-GI-NEXT:    ldp q7, q16, [x0, #32]8573; CHECK-GI-NEXT:    udot v1.4s, v5.16b, v0.16b8574; CHECK-GI-NEXT:    udot v3.4s, v6.16b, v0.16b8575; CHECK-GI-NEXT:    udot v2.4s, v16.16b, v0.16b8576; CHECK-GI-NEXT:    udot v4.4s, v7.16b, v0.16b8577; CHECK-GI-NEXT:    add v0.4s, v1.4s, v3.4s8578; CHECK-GI-NEXT:    add v1.4s, v4.4s, v2.4s8579; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s8580; CHECK-GI-NEXT:    addv s0, v0.4s8581; CHECK-GI-NEXT:    fmov w0, s08582; CHECK-GI-NEXT:    ret8583entry:8584  %0 = load <64 x i8>, ptr %a18585  %1 = zext <64 x i8> %0 to <64 x i32>8586  %2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %1)8587  ret i32 %28588}8589define i32 @test_sdot_v64i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {8590; CHECK-SD-LABEL: test_sdot_v64i8:8591; CHECK-SD:       // %bb.0: // %entry8592; CHECK-SD-NEXT:    movi v0.2d, #00000000000000008593; CHECK-SD-NEXT:    movi v1.2d, #00000000000000008594; CHECK-SD-NEXT:    ldp q2, q3, [x0, #32]8595; CHECK-SD-NEXT:    ldp q4, q5, [x1, #32]8596; CHECK-SD-NEXT:    sdot v1.4s, v5.16b, v3.16b8597; CHECK-SD-NEXT:    sdot v0.4s, v4.16b, v2.16b8598; CHECK-SD-NEXT:    ldp q2, q3, [x0]8599; CHECK-SD-NEXT:    ldp q4, q5, [x1]8600; CHECK-SD-NEXT:    sdot v1.4s, v5.16b, v3.16b8601; CHECK-SD-NEXT:    sdot v0.4s, v4.16b, v2.16b8602; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s8603; CHECK-SD-NEXT:    addv s0, v0.4s8604; CHECK-SD-NEXT:    fmov w8, s08605; CHECK-SD-NEXT:    add w0, w8, w28606; CHECK-SD-NEXT:    ret8607;8608; CHECK-GI-LABEL: test_sdot_v64i8:8609; CHECK-GI:       // %bb.0: // %entry8610; CHECK-GI-NEXT:    movi v0.2d, #00000000000000008611; CHECK-GI-NEXT:    movi v3.2d, #00000000000000008612; CHECK-GI-NEXT:    movi v4.2d, #00000000000000008613; CHECK-GI-NEXT:    movi v5.2d, #00000000000000008614; CHECK-GI-NEXT:    ldp q1, q2, [x0]8615; CHECK-GI-NEXT:    ldp q6, q7, [x0, #32]8616; CHECK-GI-NEXT:    ldp q16, q17, [x1]8617; CHECK-GI-NEXT:    ldp q18, q19, [x1, #32]8618; CHECK-GI-NEXT:    sdot v0.4s, v16.16b, v1.16b8619; CHECK-GI-NEXT:    sdot v4.4s, v17.16b, v2.16b8620; CHECK-GI-NEXT:    sdot v5.4s, v18.16b, v6.16b8621; CHECK-GI-NEXT:    sdot v3.4s, v19.16b, v7.16b8622; CHECK-GI-NEXT:    add v0.4s, v0.4s, v4.4s8623; CHECK-GI-NEXT:    add v1.4s, v5.4s, v3.4s8624; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s8625; CHECK-GI-NEXT:    addv s0, v0.4s8626; CHECK-GI-NEXT:    fmov w8, s08627; CHECK-GI-NEXT:    add w0, w8, w28628; CHECK-GI-NEXT:    ret8629entry:8630  %0 = load <64 x i8>, ptr %a8631  %1 = sext <64 x i8> %0 to <64 x i32>8632  %2 = load <64 x i8>, ptr %b8633  %3 = sext <64 x i8> %2 to <64 x i32>8634  %4 = mul nsw <64 x i32> %3, %18635  %5 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %4)8636  %op.extra = add nsw i32 %5, %sum8637  ret i32 %op.extra8638}8639 8640define i32 @test_sdot_v64i8_double(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {8641; CHECK-SD-LABEL: test_sdot_v64i8_double:8642; CHECK-SD:       // %bb.0: // %entry8643; CHECK-SD-NEXT:    movi v16.2d, #00000000000000008644; CHECK-SD-NEXT:    movi v17.2d, #00000000000000008645; CHECK-SD-NEXT:    movi v18.2d, #00000000000000008646; CHECK-SD-NEXT:    movi v19.2d, #00000000000000008647; CHECK-SD-NEXT:    ldp q20, q21, [sp, #96]8648; CHECK-SD-NEXT:    ldp q22, q23, [sp, #32]8649; CHECK-SD-NEXT:    sdot v16.4s, v3.16b, v7.16b8650; CHECK-SD-NEXT:    sdot v18.4s, v2.16b, v6.16b8651; CHECK-SD-NEXT:    sdot v19.4s, v23.16b, v21.16b8652; CHECK-SD-NEXT:    sdot v17.4s, v22.16b, v20.16b8653; CHECK-SD-NEXT:    ldp q2, q3, [sp, #64]8654; CHECK-SD-NEXT:    ldp q6, q7, [sp]8655; CHECK-SD-NEXT:    sdot v16.4s, v1.16b, v5.16b8656; CHECK-SD-NEXT:    sdot v18.4s, v0.16b, v4.16b8657; CHECK-SD-NEXT:    sdot v19.4s, v7.16b, v3.16b8658; CHECK-SD-NEXT:    sdot v17.4s, v6.16b, v2.16b8659; CHECK-SD-NEXT:    add v0.4s, v18.4s, v16.4s8660; CHECK-SD-NEXT:    add v1.4s, v17.4s, v19.4s8661; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s8662; CHECK-SD-NEXT:    addv s0, v0.4s8663; CHECK-SD-NEXT:    fmov w0, s08664; CHECK-SD-NEXT:    ret8665;8666; CHECK-GI-LABEL: test_sdot_v64i8_double:8667; CHECK-GI:       // %bb.0: // %entry8668; CHECK-GI-NEXT:    movi v18.2d, #00000000000000008669; CHECK-GI-NEXT:    movi v21.2d, #00000000000000008670; CHECK-GI-NEXT:    movi v22.2d, #00000000000000008671; CHECK-GI-NEXT:    movi v23.2d, #00000000000000008672; CHECK-GI-NEXT:    ldp q16, q17, [sp]8673; CHECK-GI-NEXT:    movi v24.2d, #00000000000000008674; CHECK-GI-NEXT:    movi v25.2d, #00000000000000008675; CHECK-GI-NEXT:    movi v26.2d, #00000000000000008676; CHECK-GI-NEXT:    movi v27.2d, #00000000000000008677; CHECK-GI-NEXT:    ldp q19, q20, [sp, #32]8678; CHECK-GI-NEXT:    sdot v18.4s, v0.16b, v4.16b8679; CHECK-GI-NEXT:    ldp q0, q4, [sp, #64]8680; CHECK-GI-NEXT:    sdot v21.4s, v1.16b, v5.16b8681; CHECK-GI-NEXT:    ldp q1, q5, [sp, #96]8682; CHECK-GI-NEXT:    sdot v22.4s, v2.16b, v6.16b8683; CHECK-GI-NEXT:    sdot v23.4s, v3.16b, v7.16b8684; CHECK-GI-NEXT:    sdot v24.4s, v16.16b, v0.16b8685; CHECK-GI-NEXT:    sdot v26.4s, v17.16b, v4.16b8686; CHECK-GI-NEXT:    sdot v27.4s, v19.16b, v1.16b8687; CHECK-GI-NEXT:    sdot v25.4s, v20.16b, v5.16b8688; CHECK-GI-NEXT:    add v0.4s, v18.4s, v21.4s8689; CHECK-GI-NEXT:    add v1.4s, v22.4s, v23.4s8690; CHECK-GI-NEXT:    add v2.4s, v24.4s, v26.4s8691; CHECK-GI-NEXT:    add v3.4s, v27.4s, v25.4s8692; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s8693; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s8694; CHECK-GI-NEXT:    addv s0, v0.4s8695; CHECK-GI-NEXT:    addv s1, v1.4s8696; CHECK-GI-NEXT:    fmov w8, s08697; CHECK-GI-NEXT:    fmov w9, s18698; CHECK-GI-NEXT:    add w0, w8, w98699; CHECK-GI-NEXT:    ret8700entry:8701  %az = sext <64 x i8> %a to <64 x i32>8702  %bz = sext <64 x i8> %b to <64 x i32>8703  %m1 = mul nuw nsw <64 x i32> %az, %bz8704  %r1 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %m1)8705  %cz = sext <64 x i8> %c to <64 x i32>8706  %dz = sext <64 x i8> %d to <64 x i32>8707  %m2 = mul nuw nsw <64 x i32> %cz, %dz8708  %r2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %m2)8709  %x = add i32 %r1, %r28710  ret i32 %x8711}8712 8713define i32 @test_sdot_v64i8_double_nomla(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {8714; CHECK-SD-LABEL: test_sdot_v64i8_double_nomla:8715; CHECK-SD:       // %bb.0: // %entry8716; CHECK-SD-NEXT:    movi v4.16b, #18717; CHECK-SD-NEXT:    movi v5.2d, #00000000000000008718; CHECK-SD-NEXT:    movi v6.2d, #00000000000000008719; CHECK-SD-NEXT:    movi v7.2d, #00000000000000008720; CHECK-SD-NEXT:    ldp q17, q18, [sp, #32]8721; CHECK-SD-NEXT:    movi v16.2d, #00000000000000008722; CHECK-SD-NEXT:    sdot v5.4s, v3.16b, v4.16b8723; CHECK-SD-NEXT:    sdot v6.4s, v17.16b, v4.16b8724; CHECK-SD-NEXT:    sdot v7.4s, v2.16b, v4.16b8725; CHECK-SD-NEXT:    ldp q2, q3, [sp]8726; CHECK-SD-NEXT:    sdot v16.4s, v18.16b, v4.16b8727; CHECK-SD-NEXT:    sdot v5.4s, v1.16b, v4.16b8728; CHECK-SD-NEXT:    sdot v6.4s, v2.16b, v4.16b8729; CHECK-SD-NEXT:    sdot v7.4s, v0.16b, v4.16b8730; CHECK-SD-NEXT:    sdot v16.4s, v3.16b, v4.16b8731; CHECK-SD-NEXT:    add v0.4s, v7.4s, v5.4s8732; CHECK-SD-NEXT:    add v1.4s, v6.4s, v16.4s8733; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s8734; CHECK-SD-NEXT:    addv s0, v0.4s8735; CHECK-SD-NEXT:    fmov w0, s08736; CHECK-SD-NEXT:    ret8737;8738; CHECK-GI-LABEL: test_sdot_v64i8_double_nomla:8739; CHECK-GI:       // %bb.0: // %entry8740; CHECK-GI-NEXT:    movi v4.16b, #18741; CHECK-GI-NEXT:    movi v5.2d, #00000000000000008742; CHECK-GI-NEXT:    movi v6.2d, #00000000000000008743; CHECK-GI-NEXT:    movi v7.2d, #00000000000000008744; CHECK-GI-NEXT:    ldp q21, q22, [sp]8745; CHECK-GI-NEXT:    movi v16.2d, #00000000000000008746; CHECK-GI-NEXT:    movi v17.2d, #00000000000000008747; CHECK-GI-NEXT:    movi v18.2d, #00000000000000008748; CHECK-GI-NEXT:    movi v19.2d, #00000000000000008749; CHECK-GI-NEXT:    movi v20.2d, #00000000000000008750; CHECK-GI-NEXT:    sdot v5.4s, v0.16b, v4.16b8751; CHECK-GI-NEXT:    sdot v6.4s, v1.16b, v4.16b8752; CHECK-GI-NEXT:    ldp q0, q1, [sp, #32]8753; CHECK-GI-NEXT:    sdot v7.4s, v2.16b, v4.16b8754; CHECK-GI-NEXT:    sdot v16.4s, v3.16b, v4.16b8755; CHECK-GI-NEXT:    sdot v17.4s, v21.16b, v4.16b8756; CHECK-GI-NEXT:    sdot v19.4s, v22.16b, v4.16b8757; CHECK-GI-NEXT:    sdot v20.4s, v0.16b, v4.16b8758; CHECK-GI-NEXT:    sdot v18.4s, v1.16b, v4.16b8759; CHECK-GI-NEXT:    add v0.4s, v5.4s, v6.4s8760; CHECK-GI-NEXT:    add v1.4s, v7.4s, v16.4s8761; CHECK-GI-NEXT:    add v2.4s, v17.4s, v19.4s8762; CHECK-GI-NEXT:    add v3.4s, v20.4s, v18.4s8763; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s8764; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s8765; CHECK-GI-NEXT:    addv s0, v0.4s8766; CHECK-GI-NEXT:    addv s1, v1.4s8767; CHECK-GI-NEXT:    fmov w8, s08768; CHECK-GI-NEXT:    fmov w9, s18769; CHECK-GI-NEXT:    add w0, w8, w98770; CHECK-GI-NEXT:    ret8771entry:8772  %az = sext <64 x i8> %a to <64 x i32>8773  %r1 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %az)8774  %cz = sext <64 x i8> %c to <64 x i32>8775  %r2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %cz)8776  %x = add i32 %r1, %r28777  ret i32 %x8778}8779 8780define i32 @test_usdot_v64i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {8781; CHECK-SD-LABEL: test_usdot_v64i8:8782; CHECK-SD:       // %bb.0: // %entry8783; CHECK-SD-NEXT:    movi v0.2d, #00000000000000008784; CHECK-SD-NEXT:    movi v3.2d, #00000000000000008785; CHECK-SD-NEXT:    movi v4.2d, #00000000000000008786; CHECK-SD-NEXT:    movi v5.2d, #00000000000000008787; CHECK-SD-NEXT:    ldp q1, q2, [x0, #32]8788; CHECK-SD-NEXT:    ldp q6, q7, [x1, #32]8789; CHECK-SD-NEXT:    ldp q16, q17, [x0]8790; CHECK-SD-NEXT:    ldp q18, q19, [x1]8791; CHECK-SD-NEXT:    usdot v0.4s, v2.16b, v7.16b8792; CHECK-SD-NEXT:    usdot v5.4s, v1.16b, v6.16b8793; CHECK-SD-NEXT:    usdot v4.4s, v17.16b, v19.16b8794; CHECK-SD-NEXT:    usdot v3.4s, v16.16b, v18.16b8795; CHECK-SD-NEXT:    add v0.4s, v4.4s, v0.4s8796; CHECK-SD-NEXT:    add v1.4s, v3.4s, v5.4s8797; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s8798; CHECK-SD-NEXT:    addv s0, v0.4s8799; CHECK-SD-NEXT:    fmov w8, s08800; CHECK-SD-NEXT:    add w0, w8, w28801; CHECK-SD-NEXT:    ret8802;8803; CHECK-GI-LABEL: test_usdot_v64i8:8804; CHECK-GI:       // %bb.0: // %entry8805; CHECK-GI-NEXT:    ldp q0, q1, [x0]8806; CHECK-GI-NEXT:    ldp q2, q5, [x1]8807; CHECK-GI-NEXT:    ldp q3, q4, [x0, #32]8808; CHECK-GI-NEXT:    ldp q6, q7, [x1, #32]8809; CHECK-GI-NEXT:    ushll v20.8h, v0.8b, #08810; CHECK-GI-NEXT:    sshll v16.8h, v2.8b, #08811; CHECK-GI-NEXT:    sshll2 v2.8h, v2.16b, #08812; CHECK-GI-NEXT:    sshll v17.8h, v5.8b, #08813; CHECK-GI-NEXT:    sshll2 v5.8h, v5.16b, #08814; CHECK-GI-NEXT:    ushll2 v0.8h, v0.16b, #08815; CHECK-GI-NEXT:    ushll v21.8h, v1.8b, #08816; CHECK-GI-NEXT:    sshll v18.8h, v6.8b, #08817; CHECK-GI-NEXT:    sshll2 v6.8h, v6.16b, #08818; CHECK-GI-NEXT:    sshll v19.8h, v7.8b, #08819; CHECK-GI-NEXT:    sshll2 v7.8h, v7.16b, #08820; CHECK-GI-NEXT:    ushll2 v1.8h, v1.16b, #08821; CHECK-GI-NEXT:    ushll v22.8h, v3.8b, #08822; CHECK-GI-NEXT:    ushll2 v3.8h, v3.16b, #08823; CHECK-GI-NEXT:    ushll v23.8h, v4.8b, #08824; CHECK-GI-NEXT:    ushll2 v4.8h, v4.16b, #08825; CHECK-GI-NEXT:    smull v24.4s, v16.4h, v20.4h8826; CHECK-GI-NEXT:    smull v25.4s, v2.4h, v0.4h8827; CHECK-GI-NEXT:    smull v26.4s, v17.4h, v21.4h8828; CHECK-GI-NEXT:    smull v27.4s, v5.4h, v1.4h8829; CHECK-GI-NEXT:    smull v28.4s, v18.4h, v22.4h8830; CHECK-GI-NEXT:    smull v29.4s, v6.4h, v3.4h8831; CHECK-GI-NEXT:    smull v30.4s, v19.4h, v23.4h8832; CHECK-GI-NEXT:    smull v31.4s, v7.4h, v4.4h8833; CHECK-GI-NEXT:    smlal2 v24.4s, v16.8h, v20.8h8834; CHECK-GI-NEXT:    smlal2 v25.4s, v2.8h, v0.8h8835; CHECK-GI-NEXT:    smlal2 v26.4s, v17.8h, v21.8h8836; CHECK-GI-NEXT:    smlal2 v27.4s, v5.8h, v1.8h8837; CHECK-GI-NEXT:    smlal2 v28.4s, v18.8h, v22.8h8838; CHECK-GI-NEXT:    smlal2 v29.4s, v6.8h, v3.8h8839; CHECK-GI-NEXT:    smlal2 v30.4s, v19.8h, v23.8h8840; CHECK-GI-NEXT:    smlal2 v31.4s, v7.8h, v4.8h8841; CHECK-GI-NEXT:    add v0.4s, v24.4s, v25.4s8842; CHECK-GI-NEXT:    add v1.4s, v26.4s, v27.4s8843; CHECK-GI-NEXT:    add v2.4s, v28.4s, v29.4s8844; CHECK-GI-NEXT:    add v3.4s, v30.4s, v31.4s8845; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s8846; CHECK-GI-NEXT:    add v1.4s, v2.4s, v3.4s8847; CHECK-GI-NEXT:    add v0.4s, v0.4s, v1.4s8848; CHECK-GI-NEXT:    addv s0, v0.4s8849; CHECK-GI-NEXT:    fmov w8, s08850; CHECK-GI-NEXT:    add w0, w8, w28851; CHECK-GI-NEXT:    ret8852entry:8853  %0 = load <64 x i8>, ptr %a8854  %1 = zext <64 x i8> %0 to <64 x i32>8855  %2 = load <64 x i8>, ptr %b8856  %3 = sext <64 x i8> %2 to <64 x i32>8857  %4 = mul nsw <64 x i32> %3, %18858  %5 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %4)8859  %op.extra = add nsw i32 %5, %sum8860  ret i32 %op.extra8861}8862 8863define i32 @test_usdot_v64i8_double(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {8864; CHECK-SD-LABEL: test_usdot_v64i8_double:8865; CHECK-SD:       // %bb.0: // %entry8866; CHECK-SD-NEXT:    movi v18.2d, #00000000000000008867; CHECK-SD-NEXT:    movi v21.2d, #00000000000000008868; CHECK-SD-NEXT:    movi v22.2d, #00000000000000008869; CHECK-SD-NEXT:    movi v23.2d, #00000000000000008870; CHECK-SD-NEXT:    ldp q16, q17, [sp, #64]8871; CHECK-SD-NEXT:    movi v24.2d, #00000000000000008872; CHECK-SD-NEXT:    movi v25.2d, #00000000000000008873; CHECK-SD-NEXT:    movi v26.2d, #00000000000000008874; CHECK-SD-NEXT:    movi v27.2d, #00000000000000008875; CHECK-SD-NEXT:    ldp q19, q20, [sp, #96]8876; CHECK-SD-NEXT:    usdot v18.4s, v3.16b, v7.16b8877; CHECK-SD-NEXT:    ldp q3, q7, [sp, #32]8878; CHECK-SD-NEXT:    usdot v21.4s, v1.16b, v5.16b8879; CHECK-SD-NEXT:    ldp q1, q5, [sp]8880; CHECK-SD-NEXT:    usdot v22.4s, v2.16b, v6.16b8881; CHECK-SD-NEXT:    usdot v23.4s, v0.16b, v4.16b8882; CHECK-SD-NEXT:    usdot v24.4s, v7.16b, v20.16b8883; CHECK-SD-NEXT:    usdot v27.4s, v3.16b, v19.16b8884; CHECK-SD-NEXT:    usdot v26.4s, v5.16b, v17.16b8885; CHECK-SD-NEXT:    usdot v25.4s, v1.16b, v16.16b8886; CHECK-SD-NEXT:    add v0.4s, v21.4s, v18.4s8887; CHECK-SD-NEXT:    add v1.4s, v23.4s, v22.4s8888; CHECK-SD-NEXT:    add v2.4s, v26.4s, v24.4s8889; CHECK-SD-NEXT:    add v3.4s, v25.4s, v27.4s8890; CHECK-SD-NEXT:    add v0.4s, v1.4s, v0.4s8891; CHECK-SD-NEXT:    add v1.4s, v3.4s, v2.4s8892; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s8893; CHECK-SD-NEXT:    addv s0, v0.4s8894; CHECK-SD-NEXT:    fmov w0, s08895; CHECK-SD-NEXT:    ret8896;8897; CHECK-GI-LABEL: test_usdot_v64i8_double:8898; CHECK-GI:       // %bb.0: // %entry8899; CHECK-GI-NEXT:    sub sp, sp, #2408900; CHECK-GI-NEXT:    stp d15, d14, [sp, #160] // 16-byte Folded Spill8901; CHECK-GI-NEXT:    stp d13, d12, [sp, #176] // 16-byte Folded Spill8902; CHECK-GI-NEXT:    stp d11, d10, [sp, #192] // 16-byte Folded Spill8903; CHECK-GI-NEXT:    stp d9, d8, [sp, #208] // 16-byte Folded Spill8904; CHECK-GI-NEXT:    str x29, [sp, #224] // 8-byte Spill8905; CHECK-GI-NEXT:    .cfi_def_cfa_offset 2408906; CHECK-GI-NEXT:    .cfi_offset w29, -168907; CHECK-GI-NEXT:    .cfi_offset b8, -248908; CHECK-GI-NEXT:    .cfi_offset b9, -328909; CHECK-GI-NEXT:    .cfi_offset b10, -408910; CHECK-GI-NEXT:    .cfi_offset b11, -488911; CHECK-GI-NEXT:    .cfi_offset b12, -568912; CHECK-GI-NEXT:    .cfi_offset b13, -648913; CHECK-GI-NEXT:    .cfi_offset b14, -728914; CHECK-GI-NEXT:    .cfi_offset b15, -808915; CHECK-GI-NEXT:    ushll v31.8h, v0.8b, #08916; CHECK-GI-NEXT:    ushll2 v8.8h, v0.16b, #08917; CHECK-GI-NEXT:    ldr x29, [sp, #224] // 8-byte Reload8918; CHECK-GI-NEXT:    sshll v11.8h, v4.8b, #08919; CHECK-GI-NEXT:    sshll2 v12.8h, v4.16b, #08920; CHECK-GI-NEXT:    ushll v9.8h, v1.8b, #08921; CHECK-GI-NEXT:    ushll2 v10.8h, v1.16b, #08922; CHECK-GI-NEXT:    ldp q25, q22, [sp, #240]8923; CHECK-GI-NEXT:    sshll v13.8h, v5.8b, #08924; CHECK-GI-NEXT:    sshll2 v14.8h, v5.16b, #08925; CHECK-GI-NEXT:    ushll v0.8h, v2.8b, #08926; CHECK-GI-NEXT:    smull v19.4s, v31.4h, v11.4h8927; CHECK-GI-NEXT:    ldp q21, q18, [sp, #272]8928; CHECK-GI-NEXT:    smull v20.4s, v8.4h, v12.4h8929; CHECK-GI-NEXT:    sshll v4.8h, v6.8b, #08930; CHECK-GI-NEXT:    sshll2 v5.8h, v6.16b, #08931; CHECK-GI-NEXT:    smull v28.4s, v9.4h, v13.4h8932; CHECK-GI-NEXT:    ldp q17, q16, [sp, #304]8933; CHECK-GI-NEXT:    smull v27.4s, v10.4h, v14.4h8934; CHECK-GI-NEXT:    sshll v6.8h, v7.8b, #08935; CHECK-GI-NEXT:    ushll2 v1.8h, v2.16b, #08936; CHECK-GI-NEXT:    smlal2 v19.4s, v31.8h, v11.8h8937; CHECK-GI-NEXT:    ldp q30, q29, [sp, #336]8938; CHECK-GI-NEXT:    smlal2 v20.4s, v8.8h, v12.8h8939; CHECK-GI-NEXT:    ushll v2.8h, v3.8b, #08940; CHECK-GI-NEXT:    ushll2 v3.8h, v3.16b, #08941; CHECK-GI-NEXT:    smlal2 v28.4s, v9.8h, v13.8h8942; CHECK-GI-NEXT:    stp q0, q6, [sp, #48] // 32-byte Folded Spill8943; CHECK-GI-NEXT:    sshll2 v7.8h, v7.16b, #08944; CHECK-GI-NEXT:    smlal2 v27.4s, v10.8h, v14.8h8945; CHECK-GI-NEXT:    smull v26.4s, v0.4h, v4.4h8946; CHECK-GI-NEXT:    ushll v31.8h, v25.8b, #08947; CHECK-GI-NEXT:    str q19, [sp, #144] // 16-byte Spill8948; CHECK-GI-NEXT:    ushll2 v25.8h, v25.16b, #08949; CHECK-GI-NEXT:    ushll v8.8h, v22.8b, #08950; CHECK-GI-NEXT:    stp q2, q1, [sp] // 32-byte Folded Spill8951; CHECK-GI-NEXT:    ushll2 v22.8h, v22.16b, #08952; CHECK-GI-NEXT:    ushll v9.8h, v21.8b, #08953; CHECK-GI-NEXT:    stp q5, q28, [sp, #80] // 32-byte Folded Spill8954; CHECK-GI-NEXT:    ushll2 v21.8h, v21.16b, #08955; CHECK-GI-NEXT:    ushll v10.8h, v18.8b, #08956; CHECK-GI-NEXT:    stp q4, q20, [sp, #112] // 32-byte Folded Spill8957; CHECK-GI-NEXT:    ushll2 v20.8h, v18.16b, #08958; CHECK-GI-NEXT:    sshll v11.8h, v17.8b, #08959; CHECK-GI-NEXT:    str q27, [sp, #32] // 16-byte Spill8960; CHECK-GI-NEXT:    ldr q28, [sp, #112] // 16-byte Reload8961; CHECK-GI-NEXT:    ldr q27, [sp, #48] // 16-byte Reload8962; CHECK-GI-NEXT:    sshll2 v19.8h, v17.16b, #08963; CHECK-GI-NEXT:    sshll v12.8h, v16.8b, #08964; CHECK-GI-NEXT:    sshll2 v18.8h, v16.16b, #08965; CHECK-GI-NEXT:    sshll v13.8h, v30.8b, #08966; CHECK-GI-NEXT:    sshll2 v30.8h, v30.16b, #08967; CHECK-GI-NEXT:    sshll v14.8h, v29.8b, #08968; CHECK-GI-NEXT:    sshll2 v29.8h, v29.16b, #08969; CHECK-GI-NEXT:    smull v23.4s, v1.4h, v5.4h8970; CHECK-GI-NEXT:    smull v15.4s, v3.4h, v7.4h8971; CHECK-GI-NEXT:    smull v24.4s, v2.4h, v6.4h8972; CHECK-GI-NEXT:    smull v17.4s, v31.4h, v11.4h8973; CHECK-GI-NEXT:    smull v6.4s, v25.4h, v19.4h8974; CHECK-GI-NEXT:    smull v16.4s, v8.4h, v12.4h8975; CHECK-GI-NEXT:    smull v4.4s, v22.4h, v18.4h8976; CHECK-GI-NEXT:    smull v5.4s, v9.4h, v13.4h8977; CHECK-GI-NEXT:    smull v2.4s, v21.4h, v30.4h8978; CHECK-GI-NEXT:    smull v1.4s, v10.4h, v14.4h8979; CHECK-GI-NEXT:    smull v0.4s, v20.4h, v29.4h8980; CHECK-GI-NEXT:    smlal2 v26.4s, v27.8h, v28.8h8981; CHECK-GI-NEXT:    ldr q28, [sp, #80] // 16-byte Reload8982; CHECK-GI-NEXT:    ldr q27, [sp, #16] // 16-byte Reload8983; CHECK-GI-NEXT:    smlal2 v15.4s, v3.8h, v7.8h8984; CHECK-GI-NEXT:    ldp q7, q3, [sp, #128] // 32-byte Folded Reload8985; CHECK-GI-NEXT:    smlal2 v23.4s, v27.8h, v28.8h8986; CHECK-GI-NEXT:    ldr q28, [sp, #64] // 16-byte Reload8987; CHECK-GI-NEXT:    ldr q27, [sp] // 16-byte Reload8988; CHECK-GI-NEXT:    smlal2 v17.4s, v31.8h, v11.8h8989; CHECK-GI-NEXT:    smlal2 v6.4s, v25.8h, v19.8h8990; CHECK-GI-NEXT:    smlal2 v16.4s, v8.8h, v12.8h8991; CHECK-GI-NEXT:    smlal2 v24.4s, v27.8h, v28.8h8992; CHECK-GI-NEXT:    smlal2 v4.4s, v22.8h, v18.8h8993; CHECK-GI-NEXT:    ldr q18, [sp, #32] // 16-byte Reload8994; CHECK-GI-NEXT:    smlal2 v5.4s, v9.8h, v13.8h8995; CHECK-GI-NEXT:    ldp d9, d8, [sp, #208] // 16-byte Folded Reload8996; CHECK-GI-NEXT:    ldp d13, d12, [sp, #176] // 16-byte Folded Reload8997; CHECK-GI-NEXT:    smlal2 v2.4s, v21.8h, v30.8h8998; CHECK-GI-NEXT:    smlal2 v1.4s, v10.8h, v14.8h8999; CHECK-GI-NEXT:    ldp d11, d10, [sp, #192] // 16-byte Folded Reload9000; CHECK-GI-NEXT:    smlal2 v0.4s, v20.8h, v29.8h9001; CHECK-GI-NEXT:    add v3.4s, v3.4s, v7.4s9002; CHECK-GI-NEXT:    ldr q7, [sp, #96] // 16-byte Reload9003; CHECK-GI-NEXT:    add v19.4s, v24.4s, v15.4s9004; CHECK-GI-NEXT:    ldp d15, d14, [sp, #160] // 16-byte Folded Reload9005; CHECK-GI-NEXT:    add v7.4s, v7.4s, v18.4s9006; CHECK-GI-NEXT:    add v18.4s, v26.4s, v23.4s9007; CHECK-GI-NEXT:    add v6.4s, v17.4s, v6.4s9008; CHECK-GI-NEXT:    add v4.4s, v16.4s, v4.4s9009; CHECK-GI-NEXT:    add v2.4s, v5.4s, v2.4s9010; CHECK-GI-NEXT:    add v0.4s, v1.4s, v0.4s9011; CHECK-GI-NEXT:    add v1.4s, v3.4s, v7.4s9012; CHECK-GI-NEXT:    add v3.4s, v18.4s, v19.4s9013; CHECK-GI-NEXT:    add v4.4s, v6.4s, v4.4s9014; CHECK-GI-NEXT:    add v0.4s, v2.4s, v0.4s9015; CHECK-GI-NEXT:    add v1.4s, v1.4s, v3.4s9016; CHECK-GI-NEXT:    add v0.4s, v4.4s, v0.4s9017; CHECK-GI-NEXT:    addv s1, v1.4s9018; CHECK-GI-NEXT:    addv s0, v0.4s9019; CHECK-GI-NEXT:    fmov w8, s19020; CHECK-GI-NEXT:    fmov w9, s09021; CHECK-GI-NEXT:    add w0, w8, w99022; CHECK-GI-NEXT:    add sp, sp, #2409023; CHECK-GI-NEXT:    ret9024entry:9025  %az = zext <64 x i8> %a to <64 x i32>9026  %bz = sext <64 x i8> %b to <64 x i32>9027  %m1 = mul nuw nsw <64 x i32> %az, %bz9028  %r1 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %m1)9029  %cz = zext <64 x i8> %c to <64 x i32>9030  %dz = sext <64 x i8> %d to <64 x i32>9031  %m2 = mul nuw nsw <64 x i32> %cz, %dz9032  %r2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %m2)9033  %x = add i32 %r1, %r29034  ret i32 %x9035}9036