brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mattr=+sve -force-streaming-compatible  < %s | FileCheck %s --check-prefixes=SVE3; RUN: llc -mattr=+sve2 -force-streaming-compatible  < %s | FileCheck %s --check-prefixes=SVE24; RUN: llc -mattr=+sme -force-streaming  < %s | FileCheck %s --check-prefixes=SVE25; RUN: llc -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE6 7; This test only tests the legal types for a given vector width, as mulh nodes8; do not get generated for non-legal types.9 10target triple = "aarch64-unknown-linux-gnu"11 12;13; SMULH14;15 16define <4 x i8> @smulh_v4i8(<4 x i8> %op1, <4 x i8> %op2) {17; SVE-LABEL: smulh_v4i8:18; SVE:       // %bb.0:19; SVE-NEXT:    ptrue p0.h, vl420; SVE-NEXT:    // kill: def $d1 killed $d1 def $z121; SVE-NEXT:    // kill: def $d0 killed $d0 def $z022; SVE-NEXT:    sxtb z0.h, p0/m, z0.h23; SVE-NEXT:    sxtb z1.h, p0/m, z1.h24; SVE-NEXT:    mul z0.h, p0/m, z0.h, z1.h25; SVE-NEXT:    lsr z0.h, z0.h, #426; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z027; SVE-NEXT:    ret28;29; SVE2-LABEL: smulh_v4i8:30; SVE2:       // %bb.0:31; SVE2-NEXT:    ptrue p0.h, vl432; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z133; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z034; SVE2-NEXT:    sxtb z0.h, p0/m, z0.h35; SVE2-NEXT:    sxtb z1.h, p0/m, z1.h36; SVE2-NEXT:    mul z0.h, z0.h, z1.h37; SVE2-NEXT:    lsr z0.h, z0.h, #438; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z039; SVE2-NEXT:    ret40;41; NONEON-NOSVE-LABEL: smulh_v4i8:42; NONEON-NOSVE:       // %bb.0:43; NONEON-NOSVE-NEXT:    sub sp, sp, #3244; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 3245; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]46; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #14]47; NONEON-NOSVE-NEXT:    ldrsb w12, [sp, #22]48; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #12]49; NONEON-NOSVE-NEXT:    ldrsb w10, [sp, #10]50; NONEON-NOSVE-NEXT:    ldrsb w11, [sp, #8]51; NONEON-NOSVE-NEXT:    ldrsb w13, [sp, #20]52; NONEON-NOSVE-NEXT:    ldrsb w14, [sp, #18]53; NONEON-NOSVE-NEXT:    mul w8, w8, w1254; NONEON-NOSVE-NEXT:    ldrsb w12, [sp, #16]55; NONEON-NOSVE-NEXT:    mul w9, w9, w1356; NONEON-NOSVE-NEXT:    mul w10, w10, w1457; NONEON-NOSVE-NEXT:    mul w11, w11, w1258; NONEON-NOSVE-NEXT:    ubfx w8, w8, #4, #1259; NONEON-NOSVE-NEXT:    ubfx w9, w9, #4, #1260; NONEON-NOSVE-NEXT:    ubfx w10, w10, #4, #1261; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]62; NONEON-NOSVE-NEXT:    ubfx w8, w11, #4, #1263; NONEON-NOSVE-NEXT:    strh w9, [sp, #28]64; NONEON-NOSVE-NEXT:    strh w10, [sp, #26]65; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]66; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]67; NONEON-NOSVE-NEXT:    add sp, sp, #3268; NONEON-NOSVE-NEXT:    ret69  %1 = sext <4 x i8> %op1 to <4 x i16>70  %2 = sext <4 x i8> %op2 to <4 x i16>71  %mul = mul <4 x i16> %1, %272  %shr = lshr <4 x i16> %mul, splat (i16 4)73  %res = trunc <4 x i16> %shr to <4 x i8>74  ret <4 x i8> %res75}76 77define <8 x i8> @smulh_v8i8(<8 x i8> %op1, <8 x i8> %op2) {78; SVE-LABEL: smulh_v8i8:79; SVE:       // %bb.0:80; SVE-NEXT:    ptrue p0.b, vl881; SVE-NEXT:    // kill: def $d0 killed $d0 def $z082; SVE-NEXT:    // kill: def $d1 killed $d1 def $z183; SVE-NEXT:    smulh z0.b, p0/m, z0.b, z1.b84; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z085; SVE-NEXT:    ret86;87; SVE2-LABEL: smulh_v8i8:88; SVE2:       // %bb.0:89; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z090; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z191; SVE2-NEXT:    smulh z0.b, z0.b, z1.b92; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z093; SVE2-NEXT:    ret94;95; NONEON-NOSVE-LABEL: smulh_v8i8:96; NONEON-NOSVE:       // %bb.0:97; NONEON-NOSVE-NEXT:    sub sp, sp, #3298; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 3299; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]100; NONEON-NOSVE-NEXT:    ldrsb w15, [sp, #15]101; NONEON-NOSVE-NEXT:    ldrsb w16, [sp, #23]102; NONEON-NOSVE-NEXT:    ldrsb w12, [sp, #12]103; NONEON-NOSVE-NEXT:    ldrsb w13, [sp, #13]104; NONEON-NOSVE-NEXT:    ldrsb w14, [sp, #14]105; NONEON-NOSVE-NEXT:    ldrsb w17, [sp, #22]106; NONEON-NOSVE-NEXT:    mul w15, w15, w16107; NONEON-NOSVE-NEXT:    ldrsb w16, [sp, #21]108; NONEON-NOSVE-NEXT:    ldrsb w18, [sp, #20]109; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #8]110; NONEON-NOSVE-NEXT:    mul w14, w14, w17111; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #9]112; NONEON-NOSVE-NEXT:    ldrsb w10, [sp, #10]113; NONEON-NOSVE-NEXT:    mul w13, w13, w16114; NONEON-NOSVE-NEXT:    ldrsb w11, [sp, #11]115; NONEON-NOSVE-NEXT:    ldrsb w17, [sp, #16]116; NONEON-NOSVE-NEXT:    mul w12, w12, w18117; NONEON-NOSVE-NEXT:    lsr w15, w15, #8118; NONEON-NOSVE-NEXT:    ldrsb w0, [sp, #19]119; NONEON-NOSVE-NEXT:    ldrsb w16, [sp, #18]120; NONEON-NOSVE-NEXT:    lsr w14, w14, #8121; NONEON-NOSVE-NEXT:    ldrsb w18, [sp, #17]122; NONEON-NOSVE-NEXT:    mul w8, w8, w17123; NONEON-NOSVE-NEXT:    lsr w13, w13, #8124; NONEON-NOSVE-NEXT:    mul w11, w11, w0125; NONEON-NOSVE-NEXT:    lsr w12, w12, #8126; NONEON-NOSVE-NEXT:    strb w15, [sp, #31]127; NONEON-NOSVE-NEXT:    mul w10, w10, w16128; NONEON-NOSVE-NEXT:    strb w14, [sp, #30]129; NONEON-NOSVE-NEXT:    mul w9, w9, w18130; NONEON-NOSVE-NEXT:    lsr w8, w8, #8131; NONEON-NOSVE-NEXT:    strb w13, [sp, #29]132; NONEON-NOSVE-NEXT:    lsr w11, w11, #8133; NONEON-NOSVE-NEXT:    strb w12, [sp, #28]134; NONEON-NOSVE-NEXT:    lsr w10, w10, #8135; NONEON-NOSVE-NEXT:    strb w8, [sp, #24]136; NONEON-NOSVE-NEXT:    lsr w9, w9, #8137; NONEON-NOSVE-NEXT:    strb w11, [sp, #27]138; NONEON-NOSVE-NEXT:    strb w10, [sp, #26]139; NONEON-NOSVE-NEXT:    strb w9, [sp, #25]140; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]141; NONEON-NOSVE-NEXT:    add sp, sp, #32142; NONEON-NOSVE-NEXT:    ret143  %1 = sext <8 x i8> %op1 to <8 x i16>144  %2 = sext <8 x i8> %op2 to <8 x i16>145  %mul = mul <8 x i16> %1, %2146  %shr = lshr <8 x i16> %mul, splat (i16 8)147  %res = trunc <8 x i16> %shr to <8 x i8>148  ret <8 x i8> %res149}150 151define <16 x i8> @smulh_v16i8(<16 x i8> %op1, <16 x i8> %op2) {152; SVE-LABEL: smulh_v16i8:153; SVE:       // %bb.0:154; SVE-NEXT:    ptrue p0.b, vl16155; SVE-NEXT:    // kill: def $q0 killed $q0 def $z0156; SVE-NEXT:    // kill: def $q1 killed $q1 def $z1157; SVE-NEXT:    smulh z0.b, p0/m, z0.b, z1.b158; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0159; SVE-NEXT:    ret160;161; SVE2-LABEL: smulh_v16i8:162; SVE2:       // %bb.0:163; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0164; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1165; SVE2-NEXT:    smulh z0.b, z0.b, z1.b166; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0167; SVE2-NEXT:    ret168;169; NONEON-NOSVE-LABEL: smulh_v16i8:170; NONEON-NOSVE:       // %bb.0:171; NONEON-NOSVE-NEXT:    sub sp, sp, #160172; NONEON-NOSVE-NEXT:    str x27, [sp, #80] // 8-byte Spill173; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #96] // 16-byte Folded Spill174; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #112] // 16-byte Folded Spill175; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #128] // 16-byte Folded Spill176; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #144] // 16-byte Folded Spill177; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 160178; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8179; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16180; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24181; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32182; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40183; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48184; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56185; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64186; NONEON-NOSVE-NEXT:    .cfi_offset w27, -80187; NONEON-NOSVE-NEXT:    str q0, [sp]188; NONEON-NOSVE-NEXT:    ldp d2, d0, [sp]189; NONEON-NOSVE-NEXT:    str q1, [sp, #16]190; NONEON-NOSVE-NEXT:    stp d0, d2, [sp, #40]191; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]192; NONEON-NOSVE-NEXT:    ldrsb w6, [sp, #44]193; NONEON-NOSVE-NEXT:    ldrsb w7, [sp, #45]194; NONEON-NOSVE-NEXT:    ldrsb w19, [sp, #46]195; NONEON-NOSVE-NEXT:    ldrsb w20, [sp, #47]196; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #48]197; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #49]198; NONEON-NOSVE-NEXT:    str d0, [sp, #56]199; NONEON-NOSVE-NEXT:    ldrsb w10, [sp, #50]200; NONEON-NOSVE-NEXT:    ldrsb w11, [sp, #51]201; NONEON-NOSVE-NEXT:    ldrsb w21, [sp, #63]202; NONEON-NOSVE-NEXT:    ldrsb w23, [sp, #62]203; NONEON-NOSVE-NEXT:    ldrsb w25, [sp, #61]204; NONEON-NOSVE-NEXT:    ldrsb w26, [sp, #60]205; NONEON-NOSVE-NEXT:    str d1, [sp, #88]206; NONEON-NOSVE-NEXT:    ldrsb w12, [sp, #52]207; NONEON-NOSVE-NEXT:    mul w20, w20, w21208; NONEON-NOSVE-NEXT:    ldrsb w13, [sp, #53]209; NONEON-NOSVE-NEXT:    ldrsb w15, [sp, #54]210; NONEON-NOSVE-NEXT:    mul w19, w19, w23211; NONEON-NOSVE-NEXT:    ldrsb w17, [sp, #55]212; NONEON-NOSVE-NEXT:    ldrsb w0, [sp, #40]213; NONEON-NOSVE-NEXT:    mul w7, w7, w25214; NONEON-NOSVE-NEXT:    ldrsb w2, [sp, #41]215; NONEON-NOSVE-NEXT:    ldrsb w3, [sp, #42]216; NONEON-NOSVE-NEXT:    mul w6, w6, w26217; NONEON-NOSVE-NEXT:    lsr w20, w20, #8218; NONEON-NOSVE-NEXT:    ldrsb w4, [sp, #43]219; NONEON-NOSVE-NEXT:    ldrsb w14, [sp, #88]220; NONEON-NOSVE-NEXT:    lsr w19, w19, #8221; NONEON-NOSVE-NEXT:    ldrsb w16, [sp, #89]222; NONEON-NOSVE-NEXT:    ldrsb w18, [sp, #90]223; NONEON-NOSVE-NEXT:    lsr w7, w7, #8224; NONEON-NOSVE-NEXT:    ldrsb w1, [sp, #91]225; NONEON-NOSVE-NEXT:    ldrsb w5, [sp, #92]226; NONEON-NOSVE-NEXT:    mul w9, w9, w16227; NONEON-NOSVE-NEXT:    lsr w6, w6, #8228; NONEON-NOSVE-NEXT:    ldrsb w22, [sp, #93]229; NONEON-NOSVE-NEXT:    ldrsb w24, [sp, #94]230; NONEON-NOSVE-NEXT:    mul w11, w11, w1231; NONEON-NOSVE-NEXT:    ldrsb w21, [sp, #95]232; NONEON-NOSVE-NEXT:    ldrsb w23, [sp, #56]233; NONEON-NOSVE-NEXT:    mul w12, w12, w5234; NONEON-NOSVE-NEXT:    ldrsb w27, [sp, #59]235; NONEON-NOSVE-NEXT:    ldrsb w25, [sp, #58]236; NONEON-NOSVE-NEXT:    mul w15, w15, w24237; NONEON-NOSVE-NEXT:    ldrsb w26, [sp, #57]238; NONEON-NOSVE-NEXT:    mul w0, w0, w23239; NONEON-NOSVE-NEXT:    lsr w11, w11, #8240; NONEON-NOSVE-NEXT:    mul w4, w4, w27241; NONEON-NOSVE-NEXT:    lsr w12, w12, #8242; NONEON-NOSVE-NEXT:    lsr w9, w9, #8243; NONEON-NOSVE-NEXT:    mul w3, w3, w25244; NONEON-NOSVE-NEXT:    lsr w15, w15, #8245; NONEON-NOSVE-NEXT:    strb w20, [sp, #79]246; NONEON-NOSVE-NEXT:    mul w2, w2, w26247; NONEON-NOSVE-NEXT:    lsr w0, w0, #8248; NONEON-NOSVE-NEXT:    strb w19, [sp, #78]249; NONEON-NOSVE-NEXT:    mul w17, w17, w21250; NONEON-NOSVE-NEXT:    lsr w4, w4, #8251; NONEON-NOSVE-NEXT:    strb w7, [sp, #77]252; NONEON-NOSVE-NEXT:    mul w13, w13, w22253; NONEON-NOSVE-NEXT:    lsr w3, w3, #8254; NONEON-NOSVE-NEXT:    strb w6, [sp, #76]255; NONEON-NOSVE-NEXT:    mul w10, w10, w18256; NONEON-NOSVE-NEXT:    lsr w2, w2, #8257; NONEON-NOSVE-NEXT:    strb w4, [sp, #75]258; NONEON-NOSVE-NEXT:    mul w8, w8, w14259; NONEON-NOSVE-NEXT:    lsr w17, w17, #8260; NONEON-NOSVE-NEXT:    strb w3, [sp, #74]261; NONEON-NOSVE-NEXT:    lsr w13, w13, #8262; NONEON-NOSVE-NEXT:    strb w2, [sp, #73]263; NONEON-NOSVE-NEXT:    ldr x27, [sp, #80] // 8-byte Reload264; NONEON-NOSVE-NEXT:    lsr w10, w10, #8265; NONEON-NOSVE-NEXT:    strb w0, [sp, #72]266; NONEON-NOSVE-NEXT:    lsr w8, w8, #8267; NONEON-NOSVE-NEXT:    strb w17, [sp, #71]268; NONEON-NOSVE-NEXT:    strb w15, [sp, #70]269; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #144] // 16-byte Folded Reload270; NONEON-NOSVE-NEXT:    strb w13, [sp, #69]271; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #128] // 16-byte Folded Reload272; NONEON-NOSVE-NEXT:    strb w12, [sp, #68]273; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #112] // 16-byte Folded Reload274; NONEON-NOSVE-NEXT:    strb w11, [sp, #67]275; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #96] // 16-byte Folded Reload276; NONEON-NOSVE-NEXT:    strb w10, [sp, #66]277; NONEON-NOSVE-NEXT:    strb w9, [sp, #65]278; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]279; NONEON-NOSVE-NEXT:    ldr q0, [sp, #64]280; NONEON-NOSVE-NEXT:    add sp, sp, #160281; NONEON-NOSVE-NEXT:    ret282  %1 = sext <16 x i8> %op1 to <16 x i16>283  %2 = sext <16 x i8> %op2 to <16 x i16>284  %mul = mul <16 x i16> %1, %2285  %shr = lshr <16 x i16> %mul, splat (i16 8)286  %res = trunc <16 x i16> %shr to <16 x i8>287  ret <16 x i8> %res288}289 290define void @smulh_v32i8(ptr %a, ptr %b) {291; SVE-LABEL: smulh_v32i8:292; SVE:       // %bb.0:293; SVE-NEXT:    ldp q0, q3, [x1]294; SVE-NEXT:    ptrue p0.b, vl16295; SVE-NEXT:    ldp q1, q2, [x0]296; SVE-NEXT:    smulh z0.b, p0/m, z0.b, z1.b297; SVE-NEXT:    smulh z2.b, p0/m, z2.b, z3.b298; SVE-NEXT:    stp q0, q2, [x0]299; SVE-NEXT:    ret300;301; SVE2-LABEL: smulh_v32i8:302; SVE2:       // %bb.0:303; SVE2-NEXT:    ldp q0, q3, [x1]304; SVE2-NEXT:    ldp q1, q2, [x0]305; SVE2-NEXT:    smulh z0.b, z1.b, z0.b306; SVE2-NEXT:    smulh z1.b, z2.b, z3.b307; SVE2-NEXT:    stp q0, q1, [x0]308; SVE2-NEXT:    ret309;310; NONEON-NOSVE-LABEL: smulh_v32i8:311; NONEON-NOSVE:       // %bb.0:312; NONEON-NOSVE-NEXT:    sub sp, sp, #384313; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #288] // 16-byte Folded Spill314; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #304] // 16-byte Folded Spill315; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #320] // 16-byte Folded Spill316; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #336] // 16-byte Folded Spill317; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #352] // 16-byte Folded Spill318; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #368] // 16-byte Folded Spill319; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 384320; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8321; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16322; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24323; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32324; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40325; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48326; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56327; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64328; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72329; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80330; NONEON-NOSVE-NEXT:    .cfi_offset w30, -88331; NONEON-NOSVE-NEXT:    .cfi_offset w29, -96332; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]333; NONEON-NOSVE-NEXT:    mov x29, x0334; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]335; NONEON-NOSVE-NEXT:    str q0, [sp, #128]336; NONEON-NOSVE-NEXT:    str q1, [sp, #160]337; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #128]338; NONEON-NOSVE-NEXT:    str q3, [sp, #144]339; NONEON-NOSVE-NEXT:    str q2, [sp, #192]340; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #176]341; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #160]342; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #184]343; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #185]344; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120] // 8-byte Folded Spill345; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #186]346; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #187]347; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #224]348; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #144]349; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112] // 8-byte Folded Spill350; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #188]351; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #189]352; NONEON-NOSVE-NEXT:    ldrsb w13, [sp, #229]353; NONEON-NOSVE-NEXT:    ldrsb w11, [sp, #227]354; NONEON-NOSVE-NEXT:    ldrsb w12, [sp, #228]355; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104] // 8-byte Folded Spill356; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #190]357; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #191]358; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #208]359; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #192]360; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #96] // 8-byte Folded Spill361; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #176]362; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #177]363; NONEON-NOSVE-NEXT:    ldrsb w10, [sp, #226]364; NONEON-NOSVE-NEXT:    ldrsb w2, [sp, #214]365; NONEON-NOSVE-NEXT:    ldrsb w1, [sp, #215]366; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #88] // 8-byte Folded Spill367; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #178]368; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #179]369; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #240]370; NONEON-NOSVE-NEXT:    ldrsb w4, [sp, #212]371; NONEON-NOSVE-NEXT:    ldrsb w3, [sp, #213]372; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #80] // 8-byte Folded Spill373; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #180]374; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #181]375; NONEON-NOSVE-NEXT:    ldrsb w14, [sp, #247]376; NONEON-NOSVE-NEXT:    ldrsb w15, [sp, #246]377; NONEON-NOSVE-NEXT:    ldrsb w16, [sp, #244]378; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #72] // 8-byte Folded Spill379; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #182]380; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #183]381; NONEON-NOSVE-NEXT:    mul w26, w12, w16382; NONEON-NOSVE-NEXT:    ldrsb w12, [sp, #242]383; NONEON-NOSVE-NEXT:    ldrsb w16, [sp, #250]384; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64] // 8-byte Folded Spill385; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #232]386; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #233]387; NONEON-NOSVE-NEXT:    mul w30, w10, w12388; NONEON-NOSVE-NEXT:    ldrsb w10, [sp, #255]389; NONEON-NOSVE-NEXT:    ldrsb w12, [sp, #253]390; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56] // 8-byte Folded Spill391; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #234]392; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #235]393; NONEON-NOSVE-NEXT:    ldrsb w0, [sp, #248]394; NONEON-NOSVE-NEXT:    ldrsb w18, [sp, #249]395; NONEON-NOSVE-NEXT:    ldrsb w6, [sp, #210]396; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48] // 8-byte Folded Spill397; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #236]398; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #237]399; NONEON-NOSVE-NEXT:    ldrsb w5, [sp, #211]400; NONEON-NOSVE-NEXT:    ldrsb w19, [sp, #208]401; NONEON-NOSVE-NEXT:    ldrsb w7, [sp, #209]402; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40] // 8-byte Folded Spill403; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #238]404; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #239]405; NONEON-NOSVE-NEXT:    ldrsb w21, [sp, #222]406; NONEON-NOSVE-NEXT:    ldrsb w20, [sp, #223]407; NONEON-NOSVE-NEXT:    ldrsb w23, [sp, #220]408; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32] // 8-byte Folded Spill409; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #224]410; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #225]411; NONEON-NOSVE-NEXT:    ldrsb w22, [sp, #221]412; NONEON-NOSVE-NEXT:    ldrsb w24, [sp, #219]413; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24] // 8-byte Folded Spill414; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #230]415; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #231]416; NONEON-NOSVE-NEXT:    mul w27, w8, w14417; NONEON-NOSVE-NEXT:    ldrsb w14, [sp, #245]418; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #217]419; NONEON-NOSVE-NEXT:    mul w9, w9, w15420; NONEON-NOSVE-NEXT:    ldrsb w15, [sp, #251]421; NONEON-NOSVE-NEXT:    mul w25, w13, w14422; NONEON-NOSVE-NEXT:    ldrsb w13, [sp, #243]423; NONEON-NOSVE-NEXT:    lsr w14, w27, #8424; NONEON-NOSVE-NEXT:    ldrsb w27, [sp, #218]425; NONEON-NOSVE-NEXT:    lsr w17, w9, #8426; NONEON-NOSVE-NEXT:    mul w28, w11, w13427; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #216]428; NONEON-NOSVE-NEXT:    strb w14, [sp, #287]429; NONEON-NOSVE-NEXT:    lsr w14, w25, #8430; NONEON-NOSVE-NEXT:    ldr w25, [sp, #24] // 4-byte Reload431; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16] // 8-byte Folded Spill432; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #241]433; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #240]434; NONEON-NOSVE-NEXT:    strb w14, [sp, #285]435; NONEON-NOSVE-NEXT:    lsr w14, w28, #8436; NONEON-NOSVE-NEXT:    ldrsb w11, [sp, #254]437; NONEON-NOSVE-NEXT:    mul w8, w25, w8438; NONEON-NOSVE-NEXT:    ldr w25, [sp, #28] // 4-byte Reload439; NONEON-NOSVE-NEXT:    ldrsb w13, [sp, #252]440; NONEON-NOSVE-NEXT:    strb w14, [sp, #283]441; NONEON-NOSVE-NEXT:    ldr w14, [sp, #40] // 4-byte Reload442; NONEON-NOSVE-NEXT:    mul w9, w25, w9443; NONEON-NOSVE-NEXT:    ldr w25, [sp, #32] // 4-byte Reload444; NONEON-NOSVE-NEXT:    strb w17, [sp, #286]445; NONEON-NOSVE-NEXT:    mul w12, w14, w12446; NONEON-NOSVE-NEXT:    lsr w8, w8, #8447; NONEON-NOSVE-NEXT:    lsr w17, w26, #8448; NONEON-NOSVE-NEXT:    mul w10, w25, w10449; NONEON-NOSVE-NEXT:    ldr w25, [sp, #36] // 4-byte Reload450; NONEON-NOSVE-NEXT:    ldr w14, [sp, #44] // 4-byte Reload451; NONEON-NOSVE-NEXT:    lsr w9, w9, #8452; NONEON-NOSVE-NEXT:    strb w8, [sp, #281]453; NONEON-NOSVE-NEXT:    mul w11, w25, w11454; NONEON-NOSVE-NEXT:    strb w17, [sp, #284]455; NONEON-NOSVE-NEXT:    lsr w17, w30, #8456; NONEON-NOSVE-NEXT:    mul w13, w14, w13457; NONEON-NOSVE-NEXT:    lsr w8, w10, #8458; NONEON-NOSVE-NEXT:    ldr w10, [sp, #48] // 4-byte Reload459; NONEON-NOSVE-NEXT:    strb w9, [sp, #280]460; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #320] // 16-byte Folded Reload461; NONEON-NOSVE-NEXT:    lsr w9, w11, #8462; NONEON-NOSVE-NEXT:    mul w10, w10, w15463; NONEON-NOSVE-NEXT:    ldr w11, [sp, #52] // 4-byte Reload464; NONEON-NOSVE-NEXT:    strb w8, [sp, #279]465; NONEON-NOSVE-NEXT:    lsr w8, w12, #8466; NONEON-NOSVE-NEXT:    ldr w12, [sp, #56] // 4-byte Reload467; NONEON-NOSVE-NEXT:    mul w11, w11, w16468; NONEON-NOSVE-NEXT:    strb w9, [sp, #278]469; NONEON-NOSVE-NEXT:    lsr w9, w13, #8470; NONEON-NOSVE-NEXT:    mul w12, w12, w18471; NONEON-NOSVE-NEXT:    ldr w13, [sp, #60] // 4-byte Reload472; NONEON-NOSVE-NEXT:    strb w8, [sp, #277]473; NONEON-NOSVE-NEXT:    lsr w8, w10, #8474; NONEON-NOSVE-NEXT:    ldr w10, [sp, #64] // 4-byte Reload475; NONEON-NOSVE-NEXT:    strb w9, [sp, #276]476; NONEON-NOSVE-NEXT:    mul w13, w13, w0477; NONEON-NOSVE-NEXT:    lsr w9, w11, #8478; NONEON-NOSVE-NEXT:    ldr w11, [sp, #68] // 4-byte Reload479; NONEON-NOSVE-NEXT:    mul w10, w10, w1480; NONEON-NOSVE-NEXT:    strb w8, [sp, #275]481; NONEON-NOSVE-NEXT:    lsr w8, w12, #8482; NONEON-NOSVE-NEXT:    mul w11, w11, w2483; NONEON-NOSVE-NEXT:    ldr w12, [sp, #72] // 4-byte Reload484; NONEON-NOSVE-NEXT:    strb w9, [sp, #274]485; NONEON-NOSVE-NEXT:    lsr w9, w13, #8486; NONEON-NOSVE-NEXT:    ldr w13, [sp, #76] // 4-byte Reload487; NONEON-NOSVE-NEXT:    strb w8, [sp, #273]488; NONEON-NOSVE-NEXT:    mul w12, w12, w3489; NONEON-NOSVE-NEXT:    lsr w8, w10, #8490; NONEON-NOSVE-NEXT:    ldr w10, [sp, #80] // 4-byte Reload491; NONEON-NOSVE-NEXT:    mul w13, w13, w4492; NONEON-NOSVE-NEXT:    strb w9, [sp, #272]493; NONEON-NOSVE-NEXT:    lsr w9, w11, #8494; NONEON-NOSVE-NEXT:    mul w10, w10, w5495; NONEON-NOSVE-NEXT:    ldr w11, [sp, #84] // 4-byte Reload496; NONEON-NOSVE-NEXT:    strb w8, [sp, #271]497; NONEON-NOSVE-NEXT:    lsr w8, w12, #8498; NONEON-NOSVE-NEXT:    ldr w12, [sp, #88] // 4-byte Reload499; NONEON-NOSVE-NEXT:    strb w9, [sp, #270]500; NONEON-NOSVE-NEXT:    mul w11, w11, w6501; NONEON-NOSVE-NEXT:    lsr w9, w13, #8502; NONEON-NOSVE-NEXT:    ldr w13, [sp, #92] // 4-byte Reload503; NONEON-NOSVE-NEXT:    mul w12, w12, w7504; NONEON-NOSVE-NEXT:    strb w8, [sp, #269]505; NONEON-NOSVE-NEXT:    lsr w8, w10, #8506; NONEON-NOSVE-NEXT:    mul w13, w13, w19507; NONEON-NOSVE-NEXT:    ldr w10, [sp, #96] // 4-byte Reload508; NONEON-NOSVE-NEXT:    strb w9, [sp, #268]509; NONEON-NOSVE-NEXT:    lsr w9, w11, #8510; NONEON-NOSVE-NEXT:    ldr w11, [sp, #100] // 4-byte Reload511; NONEON-NOSVE-NEXT:    strb w8, [sp, #267]512; NONEON-NOSVE-NEXT:    mul w10, w10, w20513; NONEON-NOSVE-NEXT:    lsr w8, w12, #8514; NONEON-NOSVE-NEXT:    ldr w12, [sp, #104] // 4-byte Reload515; NONEON-NOSVE-NEXT:    mul w11, w11, w21516; NONEON-NOSVE-NEXT:    strb w9, [sp, #266]517; NONEON-NOSVE-NEXT:    lsr w9, w13, #8518; NONEON-NOSVE-NEXT:    ldr w13, [sp, #108] // 4-byte Reload519; NONEON-NOSVE-NEXT:    mul w12, w12, w22520; NONEON-NOSVE-NEXT:    strb w8, [sp, #265]521; NONEON-NOSVE-NEXT:    lsr w8, w10, #8522; NONEON-NOSVE-NEXT:    ldr w10, [sp, #112] // 4-byte Reload523; NONEON-NOSVE-NEXT:    strb w9, [sp, #264]524; NONEON-NOSVE-NEXT:    mul w13, w13, w23525; NONEON-NOSVE-NEXT:    lsr w9, w11, #8526; NONEON-NOSVE-NEXT:    ldr w11, [sp, #116] // 4-byte Reload527; NONEON-NOSVE-NEXT:    ldp w15, w14, [sp, #16] // 8-byte Folded Reload528; NONEON-NOSVE-NEXT:    mul w10, w10, w24529; NONEON-NOSVE-NEXT:    strb w8, [sp, #263]530; NONEON-NOSVE-NEXT:    lsr w8, w12, #8531; NONEON-NOSVE-NEXT:    mul w11, w11, w27532; NONEON-NOSVE-NEXT:    ldr w12, [sp, #120] // 4-byte Reload533; NONEON-NOSVE-NEXT:    strb w9, [sp, #262]534; NONEON-NOSVE-NEXT:    lsr w9, w13, #8535; NONEON-NOSVE-NEXT:    ldr w13, [sp, #124] // 4-byte Reload536; NONEON-NOSVE-NEXT:    strb w8, [sp, #261]537; NONEON-NOSVE-NEXT:    mul w12, w12, w15538; NONEON-NOSVE-NEXT:    lsr w8, w10, #8539; NONEON-NOSVE-NEXT:    strb w17, [sp, #282]540; NONEON-NOSVE-NEXT:    mul w13, w13, w14541; NONEON-NOSVE-NEXT:    strb w9, [sp, #260]542; NONEON-NOSVE-NEXT:    lsr w9, w11, #8543; NONEON-NOSVE-NEXT:    strb w8, [sp, #259]544; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #368] // 16-byte Folded Reload545; NONEON-NOSVE-NEXT:    lsr w8, w12, #8546; NONEON-NOSVE-NEXT:    strb w9, [sp, #258]547; NONEON-NOSVE-NEXT:    lsr w9, w13, #8548; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #352] // 16-byte Folded Reload549; NONEON-NOSVE-NEXT:    strb w8, [sp, #257]550; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #336] // 16-byte Folded Reload551; NONEON-NOSVE-NEXT:    strb w9, [sp, #256]552; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #304] // 16-byte Folded Reload553; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #256]554; NONEON-NOSVE-NEXT:    stp q0, q1, [x29]555; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #288] // 16-byte Folded Reload556; NONEON-NOSVE-NEXT:    add sp, sp, #384557; NONEON-NOSVE-NEXT:    ret558  %op1 = load <32 x i8>, ptr %a559  %op2 = load <32 x i8>, ptr %b560  %1 = sext <32 x i8> %op1 to <32 x i16>561  %2 = sext <32 x i8> %op2 to <32 x i16>562  %mul = mul <32 x i16> %1, %2563  %shr = lshr <32 x i16> %mul, splat (i16 8)564  %res = trunc <32 x i16> %shr to <32 x i8>565  store <32 x i8> %res, ptr %a566  ret void567}568 569define <2 x i16> @smulh_v2i16(<2 x i16> %op1, <2 x i16> %op2) {570; SVE-LABEL: smulh_v2i16:571; SVE:       // %bb.0:572; SVE-NEXT:    ptrue p0.s, vl2573; SVE-NEXT:    // kill: def $d1 killed $d1 def $z1574; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0575; SVE-NEXT:    sxth z0.s, p0/m, z0.s576; SVE-NEXT:    sxth z1.s, p0/m, z1.s577; SVE-NEXT:    mul z0.s, p0/m, z0.s, z1.s578; SVE-NEXT:    lsr z0.s, z0.s, #16579; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z0580; SVE-NEXT:    ret581;582; SVE2-LABEL: smulh_v2i16:583; SVE2:       // %bb.0:584; SVE2-NEXT:    ptrue p0.s, vl2585; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1586; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0587; SVE2-NEXT:    sxth z0.s, p0/m, z0.s588; SVE2-NEXT:    sxth z1.s, p0/m, z1.s589; SVE2-NEXT:    mul z0.s, z0.s, z1.s590; SVE2-NEXT:    lsr z0.s, z0.s, #16591; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0592; SVE2-NEXT:    ret593;594; NONEON-NOSVE-LABEL: smulh_v2i16:595; NONEON-NOSVE:       // %bb.0:596; NONEON-NOSVE-NEXT:    sub sp, sp, #32597; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32598; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]599; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #12]600; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #8]601; NONEON-NOSVE-NEXT:    ldrsh w10, [sp, #20]602; NONEON-NOSVE-NEXT:    ldrsh w11, [sp, #16]603; NONEON-NOSVE-NEXT:    mul w8, w8, w10604; NONEON-NOSVE-NEXT:    mul w9, w9, w11605; NONEON-NOSVE-NEXT:    lsr w8, w8, #16606; NONEON-NOSVE-NEXT:    lsr w9, w9, #16607; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #24]608; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]609; NONEON-NOSVE-NEXT:    add sp, sp, #32610; NONEON-NOSVE-NEXT:    ret611  %1 = sext <2 x i16> %op1 to <2 x i32>612  %2 = sext <2 x i16> %op2 to <2 x i32>613  %mul = mul <2 x i32> %1, %2614  %shr = lshr <2 x i32> %mul, splat (i32 16)615  %res = trunc <2 x i32> %shr to <2 x i16>616  ret <2 x i16> %res617}618 619define <4 x i16> @smulh_v4i16(<4 x i16> %op1, <4 x i16> %op2) {620; SVE-LABEL: smulh_v4i16:621; SVE:       // %bb.0:622; SVE-NEXT:    ptrue p0.h, vl4623; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0624; SVE-NEXT:    // kill: def $d1 killed $d1 def $z1625; SVE-NEXT:    smulh z0.h, p0/m, z0.h, z1.h626; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z0627; SVE-NEXT:    ret628;629; SVE2-LABEL: smulh_v4i16:630; SVE2:       // %bb.0:631; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0632; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1633; SVE2-NEXT:    smulh z0.h, z0.h, z1.h634; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0635; SVE2-NEXT:    ret636;637; NONEON-NOSVE-LABEL: smulh_v4i16:638; NONEON-NOSVE:       // %bb.0:639; NONEON-NOSVE-NEXT:    sub sp, sp, #32640; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32641; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]642; NONEON-NOSVE-NEXT:    ldrsh w11, [sp, #14]643; NONEON-NOSVE-NEXT:    ldrsh w12, [sp, #22]644; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #8]645; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #10]646; NONEON-NOSVE-NEXT:    ldrsh w10, [sp, #12]647; NONEON-NOSVE-NEXT:    ldrsh w13, [sp, #20]648; NONEON-NOSVE-NEXT:    ldrsh w14, [sp, #18]649; NONEON-NOSVE-NEXT:    mul w11, w11, w12650; NONEON-NOSVE-NEXT:    ldrsh w12, [sp, #16]651; NONEON-NOSVE-NEXT:    mul w10, w10, w13652; NONEON-NOSVE-NEXT:    mul w9, w9, w14653; NONEON-NOSVE-NEXT:    mul w8, w8, w12654; NONEON-NOSVE-NEXT:    lsr w11, w11, #16655; NONEON-NOSVE-NEXT:    lsr w10, w10, #16656; NONEON-NOSVE-NEXT:    lsr w9, w9, #16657; NONEON-NOSVE-NEXT:    strh w11, [sp, #30]658; NONEON-NOSVE-NEXT:    lsr w8, w8, #16659; NONEON-NOSVE-NEXT:    strh w10, [sp, #28]660; NONEON-NOSVE-NEXT:    strh w9, [sp, #26]661; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]662; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]663; NONEON-NOSVE-NEXT:    add sp, sp, #32664; NONEON-NOSVE-NEXT:    ret665  %1 = sext <4 x i16> %op1 to <4 x i32>666  %2 = sext <4 x i16> %op2 to <4 x i32>667  %mul = mul <4 x i32> %1, %2668  %shr = lshr <4 x i32> %mul, splat (i32 16)669  %res = trunc <4 x i32> %shr to <4 x i16>670  ret <4 x i16> %res671}672 673define <8 x i16> @smulh_v8i16(<8 x i16> %op1, <8 x i16> %op2) {674; SVE-LABEL: smulh_v8i16:675; SVE:       // %bb.0:676; SVE-NEXT:    ptrue p0.h, vl8677; SVE-NEXT:    // kill: def $q0 killed $q0 def $z0678; SVE-NEXT:    // kill: def $q1 killed $q1 def $z1679; SVE-NEXT:    smulh z0.h, p0/m, z0.h, z1.h680; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0681; SVE-NEXT:    ret682;683; SVE2-LABEL: smulh_v8i16:684; SVE2:       // %bb.0:685; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0686; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1687; SVE2-NEXT:    smulh z0.h, z0.h, z1.h688; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0689; SVE2-NEXT:    ret690;691; NONEON-NOSVE-LABEL: smulh_v8i16:692; NONEON-NOSVE:       // %bb.0:693; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-80]!694; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80695; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]696; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]697; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]698; NONEON-NOSVE-NEXT:    ldrsh w15, [sp, #38]699; NONEON-NOSVE-NEXT:    ldrsh w12, [sp, #32]700; NONEON-NOSVE-NEXT:    ldrsh w13, [sp, #34]701; NONEON-NOSVE-NEXT:    ldrsh w14, [sp, #36]702; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #40]703; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #42]704; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]705; NONEON-NOSVE-NEXT:    ldrsh w10, [sp, #44]706; NONEON-NOSVE-NEXT:    ldrsh w11, [sp, #46]707; NONEON-NOSVE-NEXT:    ldrsh w16, [sp, #54]708; NONEON-NOSVE-NEXT:    ldrsh w17, [sp, #52]709; NONEON-NOSVE-NEXT:    ldrsh w18, [sp, #50]710; NONEON-NOSVE-NEXT:    ldrsh w0, [sp, #62]711; NONEON-NOSVE-NEXT:    mul w15, w15, w16712; NONEON-NOSVE-NEXT:    ldrsh w16, [sp, #48]713; NONEON-NOSVE-NEXT:    mul w14, w14, w17714; NONEON-NOSVE-NEXT:    ldrsh w17, [sp, #56]715; NONEON-NOSVE-NEXT:    mul w13, w13, w18716; NONEON-NOSVE-NEXT:    ldrsh w18, [sp, #60]717; NONEON-NOSVE-NEXT:    mul w12, w12, w16718; NONEON-NOSVE-NEXT:    ldrsh w16, [sp, #58]719; NONEON-NOSVE-NEXT:    lsr w15, w15, #16720; NONEON-NOSVE-NEXT:    mul w11, w11, w0721; NONEON-NOSVE-NEXT:    lsr w14, w14, #16722; NONEON-NOSVE-NEXT:    mul w10, w10, w18723; NONEON-NOSVE-NEXT:    lsr w13, w13, #16724; NONEON-NOSVE-NEXT:    strh w15, [sp, #78]725; NONEON-NOSVE-NEXT:    mul w9, w9, w16726; NONEON-NOSVE-NEXT:    lsr w12, w12, #16727; NONEON-NOSVE-NEXT:    strh w14, [sp, #76]728; NONEON-NOSVE-NEXT:    mul w8, w8, w17729; NONEON-NOSVE-NEXT:    lsr w11, w11, #16730; NONEON-NOSVE-NEXT:    strh w13, [sp, #74]731; NONEON-NOSVE-NEXT:    lsr w10, w10, #16732; NONEON-NOSVE-NEXT:    strh w12, [sp, #72]733; NONEON-NOSVE-NEXT:    lsr w9, w9, #16734; NONEON-NOSVE-NEXT:    strh w11, [sp, #70]735; NONEON-NOSVE-NEXT:    lsr w8, w8, #16736; NONEON-NOSVE-NEXT:    strh w10, [sp, #68]737; NONEON-NOSVE-NEXT:    strh w9, [sp, #66]738; NONEON-NOSVE-NEXT:    strh w8, [sp, #64]739; NONEON-NOSVE-NEXT:    ldr q0, [sp, #64]740; NONEON-NOSVE-NEXT:    add sp, sp, #80741; NONEON-NOSVE-NEXT:    ret742  %1 = sext <8 x i16> %op1 to <8 x i32>743  %2 = sext <8 x i16> %op2 to <8 x i32>744  %mul = mul <8 x i32> %1, %2745  %shr = lshr <8 x i32> %mul, splat (i32 16)746  %res = trunc <8 x i32> %shr to <8 x i16>747  ret <8 x i16> %res748}749 750define void @smulh_v16i16(ptr %a, ptr %b) {751; SVE-LABEL: smulh_v16i16:752; SVE:       // %bb.0:753; SVE-NEXT:    ldp q0, q3, [x1]754; SVE-NEXT:    ptrue p0.h, vl8755; SVE-NEXT:    ldp q1, q2, [x0]756; SVE-NEXT:    smulh z0.h, p0/m, z0.h, z1.h757; SVE-NEXT:    smulh z2.h, p0/m, z2.h, z3.h758; SVE-NEXT:    stp q0, q2, [x0]759; SVE-NEXT:    ret760;761; SVE2-LABEL: smulh_v16i16:762; SVE2:       // %bb.0:763; SVE2-NEXT:    ldp q0, q3, [x1]764; SVE2-NEXT:    ldp q1, q2, [x0]765; SVE2-NEXT:    smulh z0.h, z1.h, z0.h766; SVE2-NEXT:    smulh z1.h, z2.h, z3.h767; SVE2-NEXT:    stp q0, q1, [x0]768; SVE2-NEXT:    ret769;770; NONEON-NOSVE-LABEL: smulh_v16i16:771; NONEON-NOSVE:       // %bb.0:772; NONEON-NOSVE-NEXT:    sub sp, sp, #240773; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #160] // 16-byte Folded Spill774; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #176] // 16-byte Folded Spill775; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #192] // 16-byte Folded Spill776; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #208] // 16-byte Folded Spill777; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #224] // 16-byte Folded Spill778; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 240779; NONEON-NOSVE-NEXT:    .cfi_offset w19, -8780; NONEON-NOSVE-NEXT:    .cfi_offset w20, -16781; NONEON-NOSVE-NEXT:    .cfi_offset w21, -24782; NONEON-NOSVE-NEXT:    .cfi_offset w22, -32783; NONEON-NOSVE-NEXT:    .cfi_offset w23, -40784; NONEON-NOSVE-NEXT:    .cfi_offset w24, -48785; NONEON-NOSVE-NEXT:    .cfi_offset w25, -56786; NONEON-NOSVE-NEXT:    .cfi_offset w26, -64787; NONEON-NOSVE-NEXT:    .cfi_offset w27, -72788; NONEON-NOSVE-NEXT:    .cfi_offset w28, -80789; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]790; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]791; NONEON-NOSVE-NEXT:    str q0, [sp]792; NONEON-NOSVE-NEXT:    str q1, [sp, #32]793; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]794; NONEON-NOSVE-NEXT:    str q3, [sp, #16]795; NONEON-NOSVE-NEXT:    str q2, [sp, #64]796; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]797; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #32]798; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #56]799; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #58]800; NONEON-NOSVE-NEXT:    ldrsh w10, [sp, #60]801; NONEON-NOSVE-NEXT:    ldrsh w11, [sp, #62]802; NONEON-NOSVE-NEXT:    ldrsh w12, [sp, #48]803; NONEON-NOSVE-NEXT:    ldrsh w13, [sp, #50]804; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]805; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]806; NONEON-NOSVE-NEXT:    ldrsh w7, [sp, #96]807; NONEON-NOSVE-NEXT:    ldrsh w19, [sp, #98]808; NONEON-NOSVE-NEXT:    ldrsh w20, [sp, #100]809; NONEON-NOSVE-NEXT:    ldrsh w21, [sp, #102]810; NONEON-NOSVE-NEXT:    ldrsh w14, [sp, #52]811; NONEON-NOSVE-NEXT:    ldrsh w16, [sp, #54]812; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]813; NONEON-NOSVE-NEXT:    ldrsh w18, [sp, #104]814; NONEON-NOSVE-NEXT:    ldrsh w2, [sp, #106]815; NONEON-NOSVE-NEXT:    ldrsh w4, [sp, #108]816; NONEON-NOSVE-NEXT:    ldrsh w5, [sp, #110]817; NONEON-NOSVE-NEXT:    ldrsh w15, [sp, #88]818; NONEON-NOSVE-NEXT:    ldrsh w17, [sp, #90]819; NONEON-NOSVE-NEXT:    ldrsh w1, [sp, #92]820; NONEON-NOSVE-NEXT:    ldrsh w3, [sp, #94]821; NONEON-NOSVE-NEXT:    mul w8, w8, w15822; NONEON-NOSVE-NEXT:    ldrsh w6, [sp, #80]823; NONEON-NOSVE-NEXT:    ldrsh w23, [sp, #82]824; NONEON-NOSVE-NEXT:    mul w11, w11, w3825; NONEON-NOSVE-NEXT:    ldrsh w25, [sp, #84]826; NONEON-NOSVE-NEXT:    mul w13, w13, w23827; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #64]828; NONEON-NOSVE-NEXT:    mul w14, w14, w25829; NONEON-NOSVE-NEXT:    lsr w8, w8, #16830; NONEON-NOSVE-NEXT:    mul w12, w12, w6831; NONEON-NOSVE-NEXT:    lsr w11, w11, #16832; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]833; NONEON-NOSVE-NEXT:    mul w10, w10, w1834; NONEON-NOSVE-NEXT:    lsr w13, w13, #16835; NONEON-NOSVE-NEXT:    ldrsh w22, [sp, #118]836; NONEON-NOSVE-NEXT:    ldrsh w24, [sp, #116]837; NONEON-NOSVE-NEXT:    ldrsh w26, [sp, #114]838; NONEON-NOSVE-NEXT:    ldrsh w27, [sp, #112]839; NONEON-NOSVE-NEXT:    ldrsh w28, [sp, #126]840; NONEON-NOSVE-NEXT:    mul w9, w9, w17841; NONEON-NOSVE-NEXT:    mul w21, w21, w22842; NONEON-NOSVE-NEXT:    ldrsh w22, [sp, #86]843; NONEON-NOSVE-NEXT:    lsr w14, w14, #16844; NONEON-NOSVE-NEXT:    mul w20, w20, w24845; NONEON-NOSVE-NEXT:    ldrsh w24, [sp, #120]846; NONEON-NOSVE-NEXT:    lsr w12, w12, #16847; NONEON-NOSVE-NEXT:    mul w19, w19, w26848; NONEON-NOSVE-NEXT:    ldrsh w26, [sp, #124]849; NONEON-NOSVE-NEXT:    lsr w10, w10, #16850; NONEON-NOSVE-NEXT:    mul w7, w7, w27851; NONEON-NOSVE-NEXT:    ldrsh w27, [sp, #122]852; NONEON-NOSVE-NEXT:    lsr w21, w21, #16853; NONEON-NOSVE-NEXT:    mul w5, w5, w28854; NONEON-NOSVE-NEXT:    lsr w20, w20, #16855; NONEON-NOSVE-NEXT:    lsr w9, w9, #16856; NONEON-NOSVE-NEXT:    mul w4, w4, w26857; NONEON-NOSVE-NEXT:    lsr w19, w19, #16858; NONEON-NOSVE-NEXT:    strh w21, [sp, #158]859; NONEON-NOSVE-NEXT:    mul w2, w2, w27860; NONEON-NOSVE-NEXT:    lsr w7, w7, #16861; NONEON-NOSVE-NEXT:    strh w20, [sp, #156]862; NONEON-NOSVE-NEXT:    mul w18, w18, w24863; NONEON-NOSVE-NEXT:    lsr w5, w5, #16864; NONEON-NOSVE-NEXT:    strh w19, [sp, #154]865; NONEON-NOSVE-NEXT:    mul w16, w16, w22866; NONEON-NOSVE-NEXT:    lsr w4, w4, #16867; NONEON-NOSVE-NEXT:    strh w7, [sp, #152]868; NONEON-NOSVE-NEXT:    lsr w2, w2, #16869; NONEON-NOSVE-NEXT:    strh w5, [sp, #150]870; NONEON-NOSVE-NEXT:    lsr w18, w18, #16871; NONEON-NOSVE-NEXT:    strh w4, [sp, #148]872; NONEON-NOSVE-NEXT:    lsr w16, w16, #16873; NONEON-NOSVE-NEXT:    strh w2, [sp, #146]874; NONEON-NOSVE-NEXT:    strh w18, [sp, #144]875; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #224] // 16-byte Folded Reload876; NONEON-NOSVE-NEXT:    strh w16, [sp, #142]877; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #208] // 16-byte Folded Reload878; NONEON-NOSVE-NEXT:    strh w14, [sp, #140]879; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #192] // 16-byte Folded Reload880; NONEON-NOSVE-NEXT:    strh w13, [sp, #138]881; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #176] // 16-byte Folded Reload882; NONEON-NOSVE-NEXT:    strh w12, [sp, #136]883; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #160] // 16-byte Folded Reload884; NONEON-NOSVE-NEXT:    strh w11, [sp, #134]885; NONEON-NOSVE-NEXT:    strh w10, [sp, #132]886; NONEON-NOSVE-NEXT:    strh w9, [sp, #130]887; NONEON-NOSVE-NEXT:    strh w8, [sp, #128]888; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #128]889; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]890; NONEON-NOSVE-NEXT:    add sp, sp, #240891; NONEON-NOSVE-NEXT:    ret892  %op1 = load <16 x i16>, ptr %a893  %op2 = load <16 x i16>, ptr %b894  %1 = sext <16 x i16> %op1 to <16 x i32>895  %2 = sext <16 x i16> %op2 to <16 x i32>896  %mul = mul <16 x i32> %1, %2897  %shr = lshr <16 x i32> %mul, splat (i32 16)898  %res = trunc <16 x i32> %shr to <16 x i16>899  store <16 x i16> %res, ptr %a900  ret void901}902 903define <2 x i32> @smulh_v2i32(<2 x i32> %op1, <2 x i32> %op2) {904; SVE-LABEL: smulh_v2i32:905; SVE:       // %bb.0:906; SVE-NEXT:    ptrue p0.s, vl2907; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0908; SVE-NEXT:    // kill: def $d1 killed $d1 def $z1909; SVE-NEXT:    smulh z0.s, p0/m, z0.s, z1.s910; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z0911; SVE-NEXT:    ret912;913; SVE2-LABEL: smulh_v2i32:914; SVE2:       // %bb.0:915; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0916; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1917; SVE2-NEXT:    smulh z0.s, z0.s, z1.s918; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0919; SVE2-NEXT:    ret920;921; NONEON-NOSVE-LABEL: smulh_v2i32:922; NONEON-NOSVE:       // %bb.0:923; NONEON-NOSVE-NEXT:    sub sp, sp, #32924; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32925; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]926; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #8]927; NONEON-NOSVE-NEXT:    ldpsw x11, x10, [sp, #16]928; NONEON-NOSVE-NEXT:    smull x9, w9, w10929; NONEON-NOSVE-NEXT:    smull x8, w8, w11930; NONEON-NOSVE-NEXT:    lsr x9, x9, #32931; NONEON-NOSVE-NEXT:    lsr x8, x8, #32932; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]933; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]934; NONEON-NOSVE-NEXT:    add sp, sp, #32935; NONEON-NOSVE-NEXT:    ret936  %1 = sext <2 x i32> %op1 to <2 x i64>937  %2 = sext <2 x i32> %op2 to <2 x i64>938  %mul = mul <2 x i64> %1, %2939  %shr = lshr <2 x i64> %mul, splat (i64 32)940  %res = trunc <2 x i64> %shr to <2 x i32>941  ret <2 x i32> %res942}943 944define <4 x i32> @smulh_v4i32(<4 x i32> %op1, <4 x i32> %op2) {945; SVE-LABEL: smulh_v4i32:946; SVE:       // %bb.0:947; SVE-NEXT:    ptrue p0.s, vl4948; SVE-NEXT:    // kill: def $q0 killed $q0 def $z0949; SVE-NEXT:    // kill: def $q1 killed $q1 def $z1950; SVE-NEXT:    smulh z0.s, p0/m, z0.s, z1.s951; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0952; SVE-NEXT:    ret953;954; SVE2-LABEL: smulh_v4i32:955; SVE2:       // %bb.0:956; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0957; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1958; SVE2-NEXT:    smulh z0.s, z0.s, z1.s959; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0960; SVE2-NEXT:    ret961;962; NONEON-NOSVE-LABEL: smulh_v4i32:963; NONEON-NOSVE:       // %bb.0:964; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-80]!965; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80966; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]967; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]968; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]969; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #40]970; NONEON-NOSVE-NEXT:    ldpsw x10, x11, [sp, #32]971; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]972; NONEON-NOSVE-NEXT:    ldpsw x13, x12, [sp, #48]973; NONEON-NOSVE-NEXT:    smull x11, w11, w12974; NONEON-NOSVE-NEXT:    ldpsw x14, x12, [sp, #56]975; NONEON-NOSVE-NEXT:    smull x10, w10, w13976; NONEON-NOSVE-NEXT:    lsr x11, x11, #32977; NONEON-NOSVE-NEXT:    smull x9, w9, w12978; NONEON-NOSVE-NEXT:    smull x8, w8, w14979; NONEON-NOSVE-NEXT:    lsr x10, x10, #32980; NONEON-NOSVE-NEXT:    lsr x9, x9, #32981; NONEON-NOSVE-NEXT:    stp w10, w11, [sp, #72]982; NONEON-NOSVE-NEXT:    lsr x8, x8, #32983; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64]984; NONEON-NOSVE-NEXT:    ldr q0, [sp, #64]985; NONEON-NOSVE-NEXT:    add sp, sp, #80986; NONEON-NOSVE-NEXT:    ret987  %1 = sext <4 x i32> %op1 to <4 x i64>988  %2 = sext <4 x i32> %op2 to <4 x i64>989  %mul = mul <4 x i64> %1, %2990  %shr = lshr <4 x i64> %mul, splat (i64 32)991  %res = trunc <4 x i64> %shr to <4 x i32>992  ret <4 x i32> %res993}994 995define void @smulh_v8i32(ptr %a, ptr %b) {996; SVE-LABEL: smulh_v8i32:997; SVE:       // %bb.0:998; SVE-NEXT:    ldp q0, q3, [x1]999; SVE-NEXT:    ptrue p0.s, vl41000; SVE-NEXT:    ldp q1, q2, [x0]1001; SVE-NEXT:    smulh z0.s, p0/m, z0.s, z1.s1002; SVE-NEXT:    smulh z2.s, p0/m, z2.s, z3.s1003; SVE-NEXT:    stp q0, q2, [x0]1004; SVE-NEXT:    ret1005;1006; SVE2-LABEL: smulh_v8i32:1007; SVE2:       // %bb.0:1008; SVE2-NEXT:    ldp q0, q3, [x1]1009; SVE2-NEXT:    ldp q1, q2, [x0]1010; SVE2-NEXT:    smulh z0.s, z1.s, z0.s1011; SVE2-NEXT:    smulh z1.s, z2.s, z3.s1012; SVE2-NEXT:    stp q0, q1, [x0]1013; SVE2-NEXT:    ret1014;1015; NONEON-NOSVE-LABEL: smulh_v8i32:1016; NONEON-NOSVE:       // %bb.0:1017; NONEON-NOSVE-NEXT:    sub sp, sp, #1601018; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 1601019; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]1020; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]1021; NONEON-NOSVE-NEXT:    str q1, [sp, #32]1022; NONEON-NOSVE-NEXT:    stp q0, q3, [sp]1023; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]1024; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]1025; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #32]1026; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]1027; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]1028; NONEON-NOSVE-NEXT:    ldpsw x8, x9, [sp, #56]1029; NONEON-NOSVE-NEXT:    ldpsw x10, x11, [sp, #48]1030; NONEON-NOSVE-NEXT:    ldpsw x12, x13, [sp, #104]1031; NONEON-NOSVE-NEXT:    ldpsw x14, x15, [sp, #96]1032; NONEON-NOSVE-NEXT:    str q2, [sp, #64]1033; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]1034; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #64]1035; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]1036; NONEON-NOSVE-NEXT:    ldpsw x17, x16, [sp, #112]1037; NONEON-NOSVE-NEXT:    smull x15, w15, w161038; NONEON-NOSVE-NEXT:    ldpsw x18, x16, [sp, #120]1039; NONEON-NOSVE-NEXT:    smull x14, w14, w171040; NONEON-NOSVE-NEXT:    ldpsw x17, x1, [sp, #80]1041; NONEON-NOSVE-NEXT:    smull x13, w13, w161042; NONEON-NOSVE-NEXT:    lsr x15, x15, #321043; NONEON-NOSVE-NEXT:    smull x12, w12, w181044; NONEON-NOSVE-NEXT:    lsr x14, x14, #321045; NONEON-NOSVE-NEXT:    ldpsw x16, x18, [sp, #88]1046; NONEON-NOSVE-NEXT:    smull x11, w11, w11047; NONEON-NOSVE-NEXT:    lsr x13, x13, #321048; NONEON-NOSVE-NEXT:    stp w14, w15, [sp, #152]1049; NONEON-NOSVE-NEXT:    smull x10, w10, w171050; NONEON-NOSVE-NEXT:    lsr x12, x12, #321051; NONEON-NOSVE-NEXT:    smull x9, w9, w181052; NONEON-NOSVE-NEXT:    smull x8, w8, w161053; NONEON-NOSVE-NEXT:    lsr x11, x11, #321054; NONEON-NOSVE-NEXT:    stp w12, w13, [sp, #144]1055; NONEON-NOSVE-NEXT:    lsr x10, x10, #321056; NONEON-NOSVE-NEXT:    lsr x9, x9, #321057; NONEON-NOSVE-NEXT:    lsr x8, x8, #321058; NONEON-NOSVE-NEXT:    stp w10, w11, [sp, #136]1059; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]1060; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #128]1061; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]1062; NONEON-NOSVE-NEXT:    add sp, sp, #1601063; NONEON-NOSVE-NEXT:    ret1064  %op1 = load <8 x i32>, ptr %a1065  %op2 = load <8 x i32>, ptr %b1066  %1 = sext <8 x i32> %op1 to <8 x i64>1067  %2 = sext <8 x i32> %op2 to <8 x i64>1068  %mul = mul <8 x i64> %1, %21069  %shr = lshr <8 x i64> %mul,  splat (i64 32)1070  %res = trunc <8 x i64> %shr to <8 x i32>1071  store <8 x i32> %res, ptr %a1072  ret void1073}1074 1075define <1 x i64> @smulh_v1i64(<1 x i64> %op1, <1 x i64> %op2) {1076; SVE-LABEL: smulh_v1i64:1077; SVE:       // %bb.0:1078; SVE-NEXT:    ptrue p0.d, vl11079; SVE-NEXT:    // kill: def $d0 killed $d0 def $z01080; SVE-NEXT:    // kill: def $d1 killed $d1 def $z11081; SVE-NEXT:    smulh z0.d, p0/m, z0.d, z1.d1082; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z01083; SVE-NEXT:    ret1084;1085; SVE2-LABEL: smulh_v1i64:1086; SVE2:       // %bb.0:1087; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z01088; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z11089; SVE2-NEXT:    smulh z0.d, z0.d, z1.d1090; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z01091; SVE2-NEXT:    ret1092;1093; NONEON-NOSVE-LABEL: smulh_v1i64:1094; NONEON-NOSVE:       // %bb.0:1095; NONEON-NOSVE-NEXT:    sub sp, sp, #161096; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 161097; NONEON-NOSVE-NEXT:    fmov x8, d01098; NONEON-NOSVE-NEXT:    fmov x9, d11099; NONEON-NOSVE-NEXT:    smulh x8, x8, x91100; NONEON-NOSVE-NEXT:    str x8, [sp, #8]1101; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]1102; NONEON-NOSVE-NEXT:    add sp, sp, #161103; NONEON-NOSVE-NEXT:    ret1104  %1 = sext <1 x i64> %op1 to <1 x i128>1105  %2 = sext <1 x i64> %op2 to <1 x i128>1106  %mul = mul <1 x i128> %1, %21107  %shr = lshr <1 x i128> %mul, splat (i128 64)1108  %res = trunc <1 x i128> %shr to <1 x i64>1109  ret <1 x i64> %res1110}1111 1112define <2 x i64> @smulh_v2i64(<2 x i64> %op1, <2 x i64> %op2) {1113; SVE-LABEL: smulh_v2i64:1114; SVE:       // %bb.0:1115; SVE-NEXT:    ptrue p0.d, vl21116; SVE-NEXT:    // kill: def $q0 killed $q0 def $z01117; SVE-NEXT:    // kill: def $q1 killed $q1 def $z11118; SVE-NEXT:    smulh z0.d, p0/m, z0.d, z1.d1119; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z01120; SVE-NEXT:    ret1121;1122; SVE2-LABEL: smulh_v2i64:1123; SVE2:       // %bb.0:1124; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z01125; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z11126; SVE2-NEXT:    smulh z0.d, z0.d, z1.d1127; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z01128; SVE2-NEXT:    ret1129;1130; NONEON-NOSVE-LABEL: smulh_v2i64:1131; NONEON-NOSVE:       // %bb.0:1132; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-64]!1133; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 641134; NONEON-NOSVE-NEXT:    ldp x9, x8, [sp]1135; NONEON-NOSVE-NEXT:    ldp x11, x10, [sp, #16]1136; NONEON-NOSVE-NEXT:    smulh x8, x8, x101137; NONEON-NOSVE-NEXT:    smulh x9, x9, x111138; NONEON-NOSVE-NEXT:    stp x9, x8, [sp, #32]1139; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]1140; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]1141; NONEON-NOSVE-NEXT:    ldr q0, [sp, #48]1142; NONEON-NOSVE-NEXT:    add sp, sp, #641143; NONEON-NOSVE-NEXT:    ret1144  %1 = sext <2 x i64> %op1 to <2 x i128>1145  %2 = sext <2 x i64> %op2 to <2 x i128>1146  %mul = mul <2 x i128> %1, %21147  %shr = lshr <2 x i128> %mul, splat (i128 64)1148  %res = trunc <2 x i128> %shr to <2 x i64>1149  ret <2 x i64> %res1150}1151 1152define void @smulh_v4i64(ptr %a, ptr %b) {1153; SVE-LABEL: smulh_v4i64:1154; SVE:       // %bb.0:1155; SVE-NEXT:    ldp q0, q3, [x1]1156; SVE-NEXT:    ptrue p0.d, vl21157; SVE-NEXT:    ldp q1, q2, [x0]1158; SVE-NEXT:    smulh z0.d, p0/m, z0.d, z1.d1159; SVE-NEXT:    smulh z2.d, p0/m, z2.d, z3.d1160; SVE-NEXT:    stp q0, q2, [x0]1161; SVE-NEXT:    ret1162;1163; SVE2-LABEL: smulh_v4i64:1164; SVE2:       // %bb.0:1165; SVE2-NEXT:    ldp q0, q3, [x1]1166; SVE2-NEXT:    ldp q1, q2, [x0]1167; SVE2-NEXT:    smulh z0.d, z1.d, z0.d1168; SVE2-NEXT:    smulh z1.d, z2.d, z3.d1169; SVE2-NEXT:    stp q0, q1, [x0]1170; SVE2-NEXT:    ret1171;1172; NONEON-NOSVE-LABEL: smulh_v4i64:1173; NONEON-NOSVE:       // %bb.0:1174; NONEON-NOSVE-NEXT:    sub sp, sp, #1281175; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 1281176; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]1177; NONEON-NOSVE-NEXT:    ldp q2, q3, [x1]1178; NONEON-NOSVE-NEXT:    stp q1, q2, [sp]1179; NONEON-NOSVE-NEXT:    ldp x11, x10, [sp]1180; NONEON-NOSVE-NEXT:    stp q0, q3, [sp, #32]1181; NONEON-NOSVE-NEXT:    ldp x13, x12, [sp, #16]1182; NONEON-NOSVE-NEXT:    ldp x9, x8, [sp, #32]1183; NONEON-NOSVE-NEXT:    smulh x10, x10, x121184; NONEON-NOSVE-NEXT:    ldp x14, x12, [sp, #48]1185; NONEON-NOSVE-NEXT:    smulh x11, x11, x131186; NONEON-NOSVE-NEXT:    smulh x8, x8, x121187; NONEON-NOSVE-NEXT:    smulh x9, x9, x141188; NONEON-NOSVE-NEXT:    stp x11, x10, [sp, #64]1189; NONEON-NOSVE-NEXT:    stp x9, x8, [sp, #80]1190; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #80]1191; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]1192; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #64]1193; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]1194; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #96]1195; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]1196; NONEON-NOSVE-NEXT:    add sp, sp, #1281197; NONEON-NOSVE-NEXT:    ret1198  %op1 = load <4 x i64>, ptr %a1199  %op2 = load <4 x i64>, ptr %b1200  %1 = sext <4 x i64> %op1 to <4 x i128>1201  %2 = sext <4 x i64> %op2 to <4 x i128>1202  %mul = mul <4 x i128> %1, %21203  %shr = lshr <4 x i128> %mul, splat (i128 64)1204  %res = trunc <4 x i128> %shr to <4 x i64>1205  store <4 x i64> %res, ptr %a1206  ret void1207}1208 1209;1210; UMULH1211;1212 1213define <4 x i8> @umulh_v4i8(<4 x i8> %op1, <4 x i8> %op2) {1214; SVE-LABEL: umulh_v4i8:1215; SVE:       // %bb.0:1216; SVE-NEXT:    // kill: def $d1 killed $d1 def $z11217; SVE-NEXT:    // kill: def $d0 killed $d0 def $z01218; SVE-NEXT:    ptrue p0.h, vl41219; SVE-NEXT:    and z0.h, z0.h, #0xff1220; SVE-NEXT:    and z1.h, z1.h, #0xff1221; SVE-NEXT:    mul z0.h, p0/m, z0.h, z1.h1222; SVE-NEXT:    lsr z0.h, z0.h, #41223; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z01224; SVE-NEXT:    ret1225;1226; SVE2-LABEL: umulh_v4i8:1227; SVE2:       // %bb.0:1228; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z11229; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z01230; SVE2-NEXT:    and z0.h, z0.h, #0xff1231; SVE2-NEXT:    and z1.h, z1.h, #0xff1232; SVE2-NEXT:    mul z0.h, z0.h, z1.h1233; SVE2-NEXT:    lsr z0.h, z0.h, #41234; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z01235; SVE2-NEXT:    ret1236;1237; NONEON-NOSVE-LABEL: umulh_v4i8:1238; NONEON-NOSVE:       // %bb.0:1239; NONEON-NOSVE-NEXT:    sub sp, sp, #321240; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 321241; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]1242; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]1243; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #22]1244; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]1245; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #10]1246; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #8]1247; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #20]1248; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #18]1249; NONEON-NOSVE-NEXT:    mul w8, w8, w121250; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #16]1251; NONEON-NOSVE-NEXT:    mul w9, w9, w131252; NONEON-NOSVE-NEXT:    mul w10, w10, w141253; NONEON-NOSVE-NEXT:    mul w11, w11, w121254; NONEON-NOSVE-NEXT:    lsr w8, w8, #41255; NONEON-NOSVE-NEXT:    lsr w9, w9, #41256; NONEON-NOSVE-NEXT:    lsr w10, w10, #41257; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]1258; NONEON-NOSVE-NEXT:    lsr w8, w11, #41259; NONEON-NOSVE-NEXT:    strh w9, [sp, #28]1260; NONEON-NOSVE-NEXT:    strh w10, [sp, #26]1261; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]1262; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]1263; NONEON-NOSVE-NEXT:    add sp, sp, #321264; NONEON-NOSVE-NEXT:    ret1265  %1 = zext <4 x i8> %op1 to <4 x i16>1266  %2 = zext <4 x i8> %op2 to <4 x i16>1267  %mul = mul <4 x i16> %1, %21268  %shr = lshr <4 x i16> %mul, splat (i16 4)1269  %res = trunc <4 x i16> %shr to <4 x i8>1270  ret <4 x i8> %res1271}1272 1273define <8 x i8> @umulh_v8i8(<8 x i8> %op1, <8 x i8> %op2) {1274; SVE-LABEL: umulh_v8i8:1275; SVE:       // %bb.0:1276; SVE-NEXT:    ptrue p0.b, vl81277; SVE-NEXT:    // kill: def $d0 killed $d0 def $z01278; SVE-NEXT:    // kill: def $d1 killed $d1 def $z11279; SVE-NEXT:    umulh z0.b, p0/m, z0.b, z1.b1280; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z01281; SVE-NEXT:    ret1282;1283; SVE2-LABEL: umulh_v8i8:1284; SVE2:       // %bb.0:1285; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z01286; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z11287; SVE2-NEXT:    umulh z0.b, z0.b, z1.b1288; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z01289; SVE2-NEXT:    ret1290;1291; NONEON-NOSVE-LABEL: umulh_v8i8:1292; NONEON-NOSVE:       // %bb.0:1293; NONEON-NOSVE-NEXT:    sub sp, sp, #321294; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 321295; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]1296; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #15]1297; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #23]1298; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #12]1299; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #13]1300; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #14]1301; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #22]1302; NONEON-NOSVE-NEXT:    mul w15, w15, w161303; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #21]1304; NONEON-NOSVE-NEXT:    ldrb w18, [sp, #20]1305; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #8]1306; NONEON-NOSVE-NEXT:    mul w14, w14, w171307; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]1308; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #10]1309; NONEON-NOSVE-NEXT:    mul w13, w13, w161310; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #11]1311; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #16]1312; NONEON-NOSVE-NEXT:    mul w12, w12, w181313; NONEON-NOSVE-NEXT:    lsr w15, w15, #81314; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #19]1315; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #18]1316; NONEON-NOSVE-NEXT:    lsr w14, w14, #81317; NONEON-NOSVE-NEXT:    ldrb w18, [sp, #17]1318; NONEON-NOSVE-NEXT:    mul w8, w8, w171319; NONEON-NOSVE-NEXT:    lsr w13, w13, #81320; NONEON-NOSVE-NEXT:    mul w11, w11, w01321; NONEON-NOSVE-NEXT:    lsr w12, w12, #81322; NONEON-NOSVE-NEXT:    strb w15, [sp, #31]1323; NONEON-NOSVE-NEXT:    mul w10, w10, w161324; NONEON-NOSVE-NEXT:    strb w14, [sp, #30]1325; NONEON-NOSVE-NEXT:    mul w9, w9, w181326; NONEON-NOSVE-NEXT:    lsr w8, w8, #81327; NONEON-NOSVE-NEXT:    strb w13, [sp, #29]1328; NONEON-NOSVE-NEXT:    lsr w11, w11, #81329; NONEON-NOSVE-NEXT:    strb w12, [sp, #28]1330; NONEON-NOSVE-NEXT:    lsr w10, w10, #81331; NONEON-NOSVE-NEXT:    strb w8, [sp, #24]1332; NONEON-NOSVE-NEXT:    lsr w9, w9, #81333; NONEON-NOSVE-NEXT:    strb w11, [sp, #27]1334; NONEON-NOSVE-NEXT:    strb w10, [sp, #26]1335; NONEON-NOSVE-NEXT:    strb w9, [sp, #25]1336; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]1337; NONEON-NOSVE-NEXT:    add sp, sp, #321338; NONEON-NOSVE-NEXT:    ret1339  %1 = zext <8 x i8> %op1 to <8 x i16>1340  %2 = zext <8 x i8> %op2 to <8 x i16>1341  %mul = mul <8 x i16> %1, %21342  %shr = lshr <8 x i16> %mul, splat (i16 8)1343  %res = trunc <8 x i16> %shr to <8 x i8>1344  ret <8 x i8> %res1345}1346 1347define <16 x i8> @umulh_v16i8(<16 x i8> %op1, <16 x i8> %op2) {1348; SVE-LABEL: umulh_v16i8:1349; SVE:       // %bb.0:1350; SVE-NEXT:    ptrue p0.b, vl161351; SVE-NEXT:    // kill: def $q0 killed $q0 def $z01352; SVE-NEXT:    // kill: def $q1 killed $q1 def $z11353; SVE-NEXT:    umulh z0.b, p0/m, z0.b, z1.b1354; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z01355; SVE-NEXT:    ret1356;1357; SVE2-LABEL: umulh_v16i8:1358; SVE2:       // %bb.0:1359; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z01360; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z11361; SVE2-NEXT:    umulh z0.b, z0.b, z1.b1362; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z01363; SVE2-NEXT:    ret1364;1365; NONEON-NOSVE-LABEL: umulh_v16i8:1366; NONEON-NOSVE:       // %bb.0:1367; NONEON-NOSVE-NEXT:    sub sp, sp, #1601368; NONEON-NOSVE-NEXT:    str x27, [sp, #80] // 8-byte Spill1369; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #96] // 16-byte Folded Spill1370; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #112] // 16-byte Folded Spill1371; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #128] // 16-byte Folded Spill1372; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #144] // 16-byte Folded Spill1373; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 1601374; NONEON-NOSVE-NEXT:    .cfi_offset w19, -81375; NONEON-NOSVE-NEXT:    .cfi_offset w20, -161376; NONEON-NOSVE-NEXT:    .cfi_offset w21, -241377; NONEON-NOSVE-NEXT:    .cfi_offset w22, -321378; NONEON-NOSVE-NEXT:    .cfi_offset w23, -401379; NONEON-NOSVE-NEXT:    .cfi_offset w24, -481380; NONEON-NOSVE-NEXT:    .cfi_offset w25, -561381; NONEON-NOSVE-NEXT:    .cfi_offset w26, -641382; NONEON-NOSVE-NEXT:    .cfi_offset w27, -801383; NONEON-NOSVE-NEXT:    str q0, [sp]1384; NONEON-NOSVE-NEXT:    ldp d2, d0, [sp]1385; NONEON-NOSVE-NEXT:    str q1, [sp, #16]1386; NONEON-NOSVE-NEXT:    stp d0, d2, [sp, #40]1387; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]1388; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #44]1389; NONEON-NOSVE-NEXT:    ldrb w7, [sp, #45]1390; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #46]1391; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #47]1392; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #48]1393; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #49]1394; NONEON-NOSVE-NEXT:    str d0, [sp, #56]1395; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #50]1396; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #51]1397; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #63]1398; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #62]1399; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #61]1400; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #60]1401; NONEON-NOSVE-NEXT:    str d1, [sp, #88]1402; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #52]1403; NONEON-NOSVE-NEXT:    mul w20, w20, w211404; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #53]1405; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #54]1406; NONEON-NOSVE-NEXT:    mul w19, w19, w231407; NONEON-NOSVE-NEXT:    ldrb w17, [sp, #55]1408; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #40]1409; NONEON-NOSVE-NEXT:    mul w7, w7, w251410; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #41]1411; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #42]1412; NONEON-NOSVE-NEXT:    mul w6, w6, w261413; NONEON-NOSVE-NEXT:    lsr w20, w20, #81414; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #43]1415; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #88]1416; NONEON-NOSVE-NEXT:    lsr w19, w19, #81417; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #89]1418; NONEON-NOSVE-NEXT:    ldrb w18, [sp, #90]1419; NONEON-NOSVE-NEXT:    lsr w7, w7, #81420; NONEON-NOSVE-NEXT:    ldrb w1, [sp, #91]1421; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #92]1422; NONEON-NOSVE-NEXT:    mul w9, w9, w161423; NONEON-NOSVE-NEXT:    lsr w6, w6, #81424; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #93]1425; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #94]1426; NONEON-NOSVE-NEXT:    mul w11, w11, w11427; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #95]1428; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #56]1429; NONEON-NOSVE-NEXT:    mul w12, w12, w51430; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #59]1431; NONEON-NOSVE-NEXT:    ldrb w25, [sp, #58]1432; NONEON-NOSVE-NEXT:    mul w15, w15, w241433; NONEON-NOSVE-NEXT:    ldrb w26, [sp, #57]1434; NONEON-NOSVE-NEXT:    mul w0, w0, w231435; NONEON-NOSVE-NEXT:    lsr w11, w11, #81436; NONEON-NOSVE-NEXT:    mul w4, w4, w271437; NONEON-NOSVE-NEXT:    lsr w12, w12, #81438; NONEON-NOSVE-NEXT:    lsr w9, w9, #81439; NONEON-NOSVE-NEXT:    mul w3, w3, w251440; NONEON-NOSVE-NEXT:    lsr w15, w15, #81441; NONEON-NOSVE-NEXT:    strb w20, [sp, #79]1442; NONEON-NOSVE-NEXT:    mul w2, w2, w261443; NONEON-NOSVE-NEXT:    lsr w0, w0, #81444; NONEON-NOSVE-NEXT:    strb w19, [sp, #78]1445; NONEON-NOSVE-NEXT:    mul w17, w17, w211446; NONEON-NOSVE-NEXT:    lsr w4, w4, #81447; NONEON-NOSVE-NEXT:    strb w7, [sp, #77]1448; NONEON-NOSVE-NEXT:    mul w13, w13, w221449; NONEON-NOSVE-NEXT:    lsr w3, w3, #81450; NONEON-NOSVE-NEXT:    strb w6, [sp, #76]1451; NONEON-NOSVE-NEXT:    mul w10, w10, w181452; NONEON-NOSVE-NEXT:    lsr w2, w2, #81453; NONEON-NOSVE-NEXT:    strb w4, [sp, #75]1454; NONEON-NOSVE-NEXT:    mul w8, w8, w141455; NONEON-NOSVE-NEXT:    lsr w17, w17, #81456; NONEON-NOSVE-NEXT:    strb w3, [sp, #74]1457; NONEON-NOSVE-NEXT:    lsr w13, w13, #81458; NONEON-NOSVE-NEXT:    strb w2, [sp, #73]1459; NONEON-NOSVE-NEXT:    ldr x27, [sp, #80] // 8-byte Reload1460; NONEON-NOSVE-NEXT:    lsr w10, w10, #81461; NONEON-NOSVE-NEXT:    strb w0, [sp, #72]1462; NONEON-NOSVE-NEXT:    lsr w8, w8, #81463; NONEON-NOSVE-NEXT:    strb w17, [sp, #71]1464; NONEON-NOSVE-NEXT:    strb w15, [sp, #70]1465; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #144] // 16-byte Folded Reload1466; NONEON-NOSVE-NEXT:    strb w13, [sp, #69]1467; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #128] // 16-byte Folded Reload1468; NONEON-NOSVE-NEXT:    strb w12, [sp, #68]1469; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #112] // 16-byte Folded Reload1470; NONEON-NOSVE-NEXT:    strb w11, [sp, #67]1471; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #96] // 16-byte Folded Reload1472; NONEON-NOSVE-NEXT:    strb w10, [sp, #66]1473; NONEON-NOSVE-NEXT:    strb w9, [sp, #65]1474; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]1475; NONEON-NOSVE-NEXT:    ldr q0, [sp, #64]1476; NONEON-NOSVE-NEXT:    add sp, sp, #1601477; NONEON-NOSVE-NEXT:    ret1478  %1 = zext <16 x i8> %op1 to <16 x i16>1479  %2 = zext <16 x i8> %op2 to <16 x i16>1480  %mul = mul <16 x i16> %1, %21481  %shr = lshr <16 x i16> %mul, splat (i16 8)1482  %res = trunc <16 x i16> %shr to <16 x i8>1483  ret <16 x i8> %res1484}1485 1486define void @umulh_v32i8(ptr %a, ptr %b) {1487; SVE-LABEL: umulh_v32i8:1488; SVE:       // %bb.0:1489; SVE-NEXT:    ldp q0, q3, [x1]1490; SVE-NEXT:    ptrue p0.b, vl161491; SVE-NEXT:    ldp q1, q2, [x0]1492; SVE-NEXT:    umulh z0.b, p0/m, z0.b, z1.b1493; SVE-NEXT:    umulh z2.b, p0/m, z2.b, z3.b1494; SVE-NEXT:    stp q0, q2, [x0]1495; SVE-NEXT:    ret1496;1497; SVE2-LABEL: umulh_v32i8:1498; SVE2:       // %bb.0:1499; SVE2-NEXT:    ldp q0, q3, [x1]1500; SVE2-NEXT:    ldp q1, q2, [x0]1501; SVE2-NEXT:    umulh z0.b, z1.b, z0.b1502; SVE2-NEXT:    umulh z1.b, z2.b, z3.b1503; SVE2-NEXT:    stp q0, q1, [x0]1504; SVE2-NEXT:    ret1505;1506; NONEON-NOSVE-LABEL: umulh_v32i8:1507; NONEON-NOSVE:       // %bb.0:1508; NONEON-NOSVE-NEXT:    sub sp, sp, #3841509; NONEON-NOSVE-NEXT:    stp x29, x30, [sp, #288] // 16-byte Folded Spill1510; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #304] // 16-byte Folded Spill1511; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #320] // 16-byte Folded Spill1512; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #336] // 16-byte Folded Spill1513; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #352] // 16-byte Folded Spill1514; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #368] // 16-byte Folded Spill1515; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 3841516; NONEON-NOSVE-NEXT:    .cfi_offset w19, -81517; NONEON-NOSVE-NEXT:    .cfi_offset w20, -161518; NONEON-NOSVE-NEXT:    .cfi_offset w21, -241519; NONEON-NOSVE-NEXT:    .cfi_offset w22, -321520; NONEON-NOSVE-NEXT:    .cfi_offset w23, -401521; NONEON-NOSVE-NEXT:    .cfi_offset w24, -481522; NONEON-NOSVE-NEXT:    .cfi_offset w25, -561523; NONEON-NOSVE-NEXT:    .cfi_offset w26, -641524; NONEON-NOSVE-NEXT:    .cfi_offset w27, -721525; NONEON-NOSVE-NEXT:    .cfi_offset w28, -801526; NONEON-NOSVE-NEXT:    .cfi_offset w30, -881527; NONEON-NOSVE-NEXT:    .cfi_offset w29, -961528; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]1529; NONEON-NOSVE-NEXT:    mov x29, x01530; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]1531; NONEON-NOSVE-NEXT:    str q0, [sp, #128]1532; NONEON-NOSVE-NEXT:    str q1, [sp, #160]1533; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #128]1534; NONEON-NOSVE-NEXT:    str q3, [sp, #144]1535; NONEON-NOSVE-NEXT:    str q2, [sp, #192]1536; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #176]1537; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #160]1538; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #184]1539; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #185]1540; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #120] // 8-byte Folded Spill1541; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #186]1542; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #187]1543; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #224]1544; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #144]1545; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #112] // 8-byte Folded Spill1546; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #188]1547; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #189]1548; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #229]1549; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #227]1550; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #228]1551; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #104] // 8-byte Folded Spill1552; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #190]1553; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #191]1554; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #208]1555; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #192]1556; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #96] // 8-byte Folded Spill1557; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #176]1558; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #177]1559; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #226]1560; NONEON-NOSVE-NEXT:    ldrb w2, [sp, #214]1561; NONEON-NOSVE-NEXT:    ldrb w1, [sp, #215]1562; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #88] // 8-byte Folded Spill1563; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #178]1564; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #179]1565; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #240]1566; NONEON-NOSVE-NEXT:    ldrb w4, [sp, #212]1567; NONEON-NOSVE-NEXT:    ldrb w3, [sp, #213]1568; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #80] // 8-byte Folded Spill1569; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #180]1570; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #181]1571; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #247]1572; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #246]1573; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #244]1574; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #72] // 8-byte Folded Spill1575; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #182]1576; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #183]1577; NONEON-NOSVE-NEXT:    mul w26, w12, w161578; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #242]1579; NONEON-NOSVE-NEXT:    ldrb w16, [sp, #250]1580; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64] // 8-byte Folded Spill1581; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #232]1582; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #233]1583; NONEON-NOSVE-NEXT:    mul w30, w10, w121584; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #255]1585; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #253]1586; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56] // 8-byte Folded Spill1587; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #234]1588; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #235]1589; NONEON-NOSVE-NEXT:    ldrb w0, [sp, #248]1590; NONEON-NOSVE-NEXT:    ldrb w18, [sp, #249]1591; NONEON-NOSVE-NEXT:    ldrb w6, [sp, #210]1592; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48] // 8-byte Folded Spill1593; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #236]1594; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #237]1595; NONEON-NOSVE-NEXT:    ldrb w5, [sp, #211]1596; NONEON-NOSVE-NEXT:    ldrb w19, [sp, #208]1597; NONEON-NOSVE-NEXT:    ldrb w7, [sp, #209]1598; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40] // 8-byte Folded Spill1599; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #238]1600; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #239]1601; NONEON-NOSVE-NEXT:    ldrb w21, [sp, #222]1602; NONEON-NOSVE-NEXT:    ldrb w20, [sp, #223]1603; NONEON-NOSVE-NEXT:    ldrb w23, [sp, #220]1604; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32] // 8-byte Folded Spill1605; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #224]1606; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #225]1607; NONEON-NOSVE-NEXT:    ldrb w22, [sp, #221]1608; NONEON-NOSVE-NEXT:    ldrb w24, [sp, #219]1609; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24] // 8-byte Folded Spill1610; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #230]1611; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #231]1612; NONEON-NOSVE-NEXT:    mul w27, w8, w141613; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #245]1614; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #217]1615; NONEON-NOSVE-NEXT:    mul w9, w9, w151616; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #251]1617; NONEON-NOSVE-NEXT:    mul w25, w13, w141618; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #243]1619; NONEON-NOSVE-NEXT:    lsr w14, w27, #81620; NONEON-NOSVE-NEXT:    ldrb w27, [sp, #218]1621; NONEON-NOSVE-NEXT:    lsr w17, w9, #81622; NONEON-NOSVE-NEXT:    mul w28, w11, w131623; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #216]1624; NONEON-NOSVE-NEXT:    strb w14, [sp, #287]1625; NONEON-NOSVE-NEXT:    lsr w14, w25, #81626; NONEON-NOSVE-NEXT:    ldr w25, [sp, #24] // 4-byte Reload1627; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16] // 8-byte Folded Spill1628; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #241]1629; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #240]1630; NONEON-NOSVE-NEXT:    strb w14, [sp, #285]1631; NONEON-NOSVE-NEXT:    lsr w14, w28, #81632; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #254]1633; NONEON-NOSVE-NEXT:    mul w8, w25, w81634; NONEON-NOSVE-NEXT:    ldr w25, [sp, #28] // 4-byte Reload1635; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #252]1636; NONEON-NOSVE-NEXT:    strb w14, [sp, #283]1637; NONEON-NOSVE-NEXT:    ldr w14, [sp, #40] // 4-byte Reload1638; NONEON-NOSVE-NEXT:    mul w9, w25, w91639; NONEON-NOSVE-NEXT:    ldr w25, [sp, #32] // 4-byte Reload1640; NONEON-NOSVE-NEXT:    strb w17, [sp, #286]1641; NONEON-NOSVE-NEXT:    mul w12, w14, w121642; NONEON-NOSVE-NEXT:    lsr w8, w8, #81643; NONEON-NOSVE-NEXT:    lsr w17, w26, #81644; NONEON-NOSVE-NEXT:    mul w10, w25, w101645; NONEON-NOSVE-NEXT:    ldr w25, [sp, #36] // 4-byte Reload1646; NONEON-NOSVE-NEXT:    ldr w14, [sp, #44] // 4-byte Reload1647; NONEON-NOSVE-NEXT:    lsr w9, w9, #81648; NONEON-NOSVE-NEXT:    strb w8, [sp, #281]1649; NONEON-NOSVE-NEXT:    mul w11, w25, w111650; NONEON-NOSVE-NEXT:    strb w17, [sp, #284]1651; NONEON-NOSVE-NEXT:    lsr w17, w30, #81652; NONEON-NOSVE-NEXT:    mul w13, w14, w131653; NONEON-NOSVE-NEXT:    lsr w8, w10, #81654; NONEON-NOSVE-NEXT:    ldr w10, [sp, #48] // 4-byte Reload1655; NONEON-NOSVE-NEXT:    strb w9, [sp, #280]1656; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #320] // 16-byte Folded Reload1657; NONEON-NOSVE-NEXT:    lsr w9, w11, #81658; NONEON-NOSVE-NEXT:    mul w10, w10, w151659; NONEON-NOSVE-NEXT:    ldr w11, [sp, #52] // 4-byte Reload1660; NONEON-NOSVE-NEXT:    strb w8, [sp, #279]1661; NONEON-NOSVE-NEXT:    lsr w8, w12, #81662; NONEON-NOSVE-NEXT:    ldr w12, [sp, #56] // 4-byte Reload1663; NONEON-NOSVE-NEXT:    mul w11, w11, w161664; NONEON-NOSVE-NEXT:    strb w9, [sp, #278]1665; NONEON-NOSVE-NEXT:    lsr w9, w13, #81666; NONEON-NOSVE-NEXT:    mul w12, w12, w181667; NONEON-NOSVE-NEXT:    ldr w13, [sp, #60] // 4-byte Reload1668; NONEON-NOSVE-NEXT:    strb w8, [sp, #277]1669; NONEON-NOSVE-NEXT:    lsr w8, w10, #81670; NONEON-NOSVE-NEXT:    ldr w10, [sp, #64] // 4-byte Reload1671; NONEON-NOSVE-NEXT:    strb w9, [sp, #276]1672; NONEON-NOSVE-NEXT:    mul w13, w13, w01673; NONEON-NOSVE-NEXT:    lsr w9, w11, #81674; NONEON-NOSVE-NEXT:    ldr w11, [sp, #68] // 4-byte Reload1675; NONEON-NOSVE-NEXT:    mul w10, w10, w11676; NONEON-NOSVE-NEXT:    strb w8, [sp, #275]1677; NONEON-NOSVE-NEXT:    lsr w8, w12, #81678; NONEON-NOSVE-NEXT:    mul w11, w11, w21679; NONEON-NOSVE-NEXT:    ldr w12, [sp, #72] // 4-byte Reload1680; NONEON-NOSVE-NEXT:    strb w9, [sp, #274]1681; NONEON-NOSVE-NEXT:    lsr w9, w13, #81682; NONEON-NOSVE-NEXT:    ldr w13, [sp, #76] // 4-byte Reload1683; NONEON-NOSVE-NEXT:    strb w8, [sp, #273]1684; NONEON-NOSVE-NEXT:    mul w12, w12, w31685; NONEON-NOSVE-NEXT:    lsr w8, w10, #81686; NONEON-NOSVE-NEXT:    ldr w10, [sp, #80] // 4-byte Reload1687; NONEON-NOSVE-NEXT:    mul w13, w13, w41688; NONEON-NOSVE-NEXT:    strb w9, [sp, #272]1689; NONEON-NOSVE-NEXT:    lsr w9, w11, #81690; NONEON-NOSVE-NEXT:    mul w10, w10, w51691; NONEON-NOSVE-NEXT:    ldr w11, [sp, #84] // 4-byte Reload1692; NONEON-NOSVE-NEXT:    strb w8, [sp, #271]1693; NONEON-NOSVE-NEXT:    lsr w8, w12, #81694; NONEON-NOSVE-NEXT:    ldr w12, [sp, #88] // 4-byte Reload1695; NONEON-NOSVE-NEXT:    strb w9, [sp, #270]1696; NONEON-NOSVE-NEXT:    mul w11, w11, w61697; NONEON-NOSVE-NEXT:    lsr w9, w13, #81698; NONEON-NOSVE-NEXT:    ldr w13, [sp, #92] // 4-byte Reload1699; NONEON-NOSVE-NEXT:    mul w12, w12, w71700; NONEON-NOSVE-NEXT:    strb w8, [sp, #269]1701; NONEON-NOSVE-NEXT:    lsr w8, w10, #81702; NONEON-NOSVE-NEXT:    mul w13, w13, w191703; NONEON-NOSVE-NEXT:    ldr w10, [sp, #96] // 4-byte Reload1704; NONEON-NOSVE-NEXT:    strb w9, [sp, #268]1705; NONEON-NOSVE-NEXT:    lsr w9, w11, #81706; NONEON-NOSVE-NEXT:    ldr w11, [sp, #100] // 4-byte Reload1707; NONEON-NOSVE-NEXT:    strb w8, [sp, #267]1708; NONEON-NOSVE-NEXT:    mul w10, w10, w201709; NONEON-NOSVE-NEXT:    lsr w8, w12, #81710; NONEON-NOSVE-NEXT:    ldr w12, [sp, #104] // 4-byte Reload1711; NONEON-NOSVE-NEXT:    mul w11, w11, w211712; NONEON-NOSVE-NEXT:    strb w9, [sp, #266]1713; NONEON-NOSVE-NEXT:    lsr w9, w13, #81714; NONEON-NOSVE-NEXT:    ldr w13, [sp, #108] // 4-byte Reload1715; NONEON-NOSVE-NEXT:    mul w12, w12, w221716; NONEON-NOSVE-NEXT:    strb w8, [sp, #265]1717; NONEON-NOSVE-NEXT:    lsr w8, w10, #81718; NONEON-NOSVE-NEXT:    ldr w10, [sp, #112] // 4-byte Reload1719; NONEON-NOSVE-NEXT:    strb w9, [sp, #264]1720; NONEON-NOSVE-NEXT:    mul w13, w13, w231721; NONEON-NOSVE-NEXT:    lsr w9, w11, #81722; NONEON-NOSVE-NEXT:    ldr w11, [sp, #116] // 4-byte Reload1723; NONEON-NOSVE-NEXT:    ldp w15, w14, [sp, #16] // 8-byte Folded Reload1724; NONEON-NOSVE-NEXT:    mul w10, w10, w241725; NONEON-NOSVE-NEXT:    strb w8, [sp, #263]1726; NONEON-NOSVE-NEXT:    lsr w8, w12, #81727; NONEON-NOSVE-NEXT:    mul w11, w11, w271728; NONEON-NOSVE-NEXT:    ldr w12, [sp, #120] // 4-byte Reload1729; NONEON-NOSVE-NEXT:    strb w9, [sp, #262]1730; NONEON-NOSVE-NEXT:    lsr w9, w13, #81731; NONEON-NOSVE-NEXT:    ldr w13, [sp, #124] // 4-byte Reload1732; NONEON-NOSVE-NEXT:    strb w8, [sp, #261]1733; NONEON-NOSVE-NEXT:    mul w12, w12, w151734; NONEON-NOSVE-NEXT:    lsr w8, w10, #81735; NONEON-NOSVE-NEXT:    strb w17, [sp, #282]1736; NONEON-NOSVE-NEXT:    mul w13, w13, w141737; NONEON-NOSVE-NEXT:    strb w9, [sp, #260]1738; NONEON-NOSVE-NEXT:    lsr w9, w11, #81739; NONEON-NOSVE-NEXT:    strb w8, [sp, #259]1740; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #368] // 16-byte Folded Reload1741; NONEON-NOSVE-NEXT:    lsr w8, w12, #81742; NONEON-NOSVE-NEXT:    strb w9, [sp, #258]1743; NONEON-NOSVE-NEXT:    lsr w9, w13, #81744; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #352] // 16-byte Folded Reload1745; NONEON-NOSVE-NEXT:    strb w8, [sp, #257]1746; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #336] // 16-byte Folded Reload1747; NONEON-NOSVE-NEXT:    strb w9, [sp, #256]1748; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #304] // 16-byte Folded Reload1749; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #256]1750; NONEON-NOSVE-NEXT:    stp q0, q1, [x29]1751; NONEON-NOSVE-NEXT:    ldp x29, x30, [sp, #288] // 16-byte Folded Reload1752; NONEON-NOSVE-NEXT:    add sp, sp, #3841753; NONEON-NOSVE-NEXT:    ret1754  %op1 = load <32 x i8>, ptr %a1755  %op2 = load <32 x i8>, ptr %b1756  %1 = zext <32 x i8> %op1 to <32 x i16>1757  %2 = zext <32 x i8> %op2 to <32 x i16>1758  %mul = mul <32 x i16> %1, %21759  %shr = lshr <32 x i16> %mul, splat (i16 8)1760  %res = trunc <32 x i16> %shr to <32 x i8>1761  store <32 x i8> %res, ptr %a1762  ret void1763}1764 1765define <2 x i16> @umulh_v2i16(<2 x i16> %op1, <2 x i16> %op2) {1766; SVE-LABEL: umulh_v2i16:1767; SVE:       // %bb.0:1768; SVE-NEXT:    // kill: def $d1 killed $d1 def $z11769; SVE-NEXT:    // kill: def $d0 killed $d0 def $z01770; SVE-NEXT:    ptrue p0.s, vl21771; SVE-NEXT:    and z0.s, z0.s, #0xffff1772; SVE-NEXT:    and z1.s, z1.s, #0xffff1773; SVE-NEXT:    mul z0.s, p0/m, z0.s, z1.s1774; SVE-NEXT:    lsr z0.s, z0.s, #161775; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z01776; SVE-NEXT:    ret1777;1778; SVE2-LABEL: umulh_v2i16:1779; SVE2:       // %bb.0:1780; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z11781; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z01782; SVE2-NEXT:    and z0.s, z0.s, #0xffff1783; SVE2-NEXT:    and z1.s, z1.s, #0xffff1784; SVE2-NEXT:    mul z0.s, z0.s, z1.s1785; SVE2-NEXT:    lsr z0.s, z0.s, #161786; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z01787; SVE2-NEXT:    ret1788;1789; NONEON-NOSVE-LABEL: umulh_v2i16:1790; NONEON-NOSVE:       // %bb.0:1791; NONEON-NOSVE-NEXT:    sub sp, sp, #321792; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 321793; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]1794; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]1795; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]1796; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #20]1797; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #16]1798; NONEON-NOSVE-NEXT:    mul w8, w8, w101799; NONEON-NOSVE-NEXT:    mul w9, w9, w111800; NONEON-NOSVE-NEXT:    lsr w8, w8, #161801; NONEON-NOSVE-NEXT:    lsr w9, w9, #161802; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #24]1803; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]1804; NONEON-NOSVE-NEXT:    add sp, sp, #321805; NONEON-NOSVE-NEXT:    ret1806  %1 = zext <2 x i16> %op1 to <2 x i32>1807  %2 = zext <2 x i16> %op2 to <2 x i32>1808  %mul = mul <2 x i32> %1, %21809  %shr = lshr <2 x i32> %mul, splat (i32 16)1810  %res = trunc <2 x i32> %shr to <2 x i16>1811  ret <2 x i16> %res1812}1813 1814define <4 x i16> @umulh_v4i16(<4 x i16> %op1, <4 x i16> %op2) {1815; SVE-LABEL: umulh_v4i16:1816; SVE:       // %bb.0:1817; SVE-NEXT:    ptrue p0.h, vl41818; SVE-NEXT:    // kill: def $d0 killed $d0 def $z01819; SVE-NEXT:    // kill: def $d1 killed $d1 def $z11820; SVE-NEXT:    umulh z0.h, p0/m, z0.h, z1.h1821; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z01822; SVE-NEXT:    ret1823;1824; SVE2-LABEL: umulh_v4i16:1825; SVE2:       // %bb.0:1826; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z01827; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z11828; SVE2-NEXT:    umulh z0.h, z0.h, z1.h1829; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z01830; SVE2-NEXT:    ret1831;1832; NONEON-NOSVE-LABEL: umulh_v4i16:1833; NONEON-NOSVE:       // %bb.0:1834; NONEON-NOSVE-NEXT:    sub sp, sp, #321835; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 321836; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]1837; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #14]1838; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #22]1839; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #8]1840; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]1841; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #12]1842; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #20]1843; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #18]1844; NONEON-NOSVE-NEXT:    mul w11, w11, w121845; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #16]1846; NONEON-NOSVE-NEXT:    mul w10, w10, w131847; NONEON-NOSVE-NEXT:    mul w9, w9, w141848; NONEON-NOSVE-NEXT:    mul w8, w8, w121849; NONEON-NOSVE-NEXT:    lsr w11, w11, #161850; NONEON-NOSVE-NEXT:    lsr w10, w10, #161851; NONEON-NOSVE-NEXT:    lsr w9, w9, #161852; NONEON-NOSVE-NEXT:    strh w11, [sp, #30]1853; NONEON-NOSVE-NEXT:    lsr w8, w8, #161854; NONEON-NOSVE-NEXT:    strh w10, [sp, #28]1855; NONEON-NOSVE-NEXT:    strh w9, [sp, #26]1856; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]1857; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]1858; NONEON-NOSVE-NEXT:    add sp, sp, #321859; NONEON-NOSVE-NEXT:    ret1860  %1 = zext <4 x i16> %op1 to <4 x i32>1861  %2 = zext <4 x i16> %op2 to <4 x i32>1862  %mul = mul <4 x i32> %1, %21863  %shr = lshr <4 x i32> %mul, splat (i32 16)1864  %res = trunc <4 x i32> %shr to <4 x i16>1865  ret <4 x i16> %res1866}1867 1868define <8 x i16> @umulh_v8i16(<8 x i16> %op1, <8 x i16> %op2) {1869; SVE-LABEL: umulh_v8i16:1870; SVE:       // %bb.0:1871; SVE-NEXT:    ptrue p0.h, vl81872; SVE-NEXT:    // kill: def $q0 killed $q0 def $z01873; SVE-NEXT:    // kill: def $q1 killed $q1 def $z11874; SVE-NEXT:    umulh z0.h, p0/m, z0.h, z1.h1875; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z01876; SVE-NEXT:    ret1877;1878; SVE2-LABEL: umulh_v8i16:1879; SVE2:       // %bb.0:1880; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z01881; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z11882; SVE2-NEXT:    umulh z0.h, z0.h, z1.h1883; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z01884; SVE2-NEXT:    ret1885;1886; NONEON-NOSVE-LABEL: umulh_v8i16:1887; NONEON-NOSVE:       // %bb.0:1888; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-80]!1889; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 801890; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]1891; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]1892; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]1893; NONEON-NOSVE-NEXT:    ldrh w15, [sp, #38]1894; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #32]1895; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #34]1896; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #36]1897; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #40]1898; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #42]1899; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]1900; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #44]1901; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #46]1902; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #54]1903; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #52]1904; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #50]1905; NONEON-NOSVE-NEXT:    ldrh w0, [sp, #62]1906; NONEON-NOSVE-NEXT:    mul w15, w15, w161907; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #48]1908; NONEON-NOSVE-NEXT:    mul w14, w14, w171909; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #56]1910; NONEON-NOSVE-NEXT:    mul w13, w13, w181911; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #60]1912; NONEON-NOSVE-NEXT:    mul w12, w12, w161913; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #58]1914; NONEON-NOSVE-NEXT:    lsr w15, w15, #161915; NONEON-NOSVE-NEXT:    mul w11, w11, w01916; NONEON-NOSVE-NEXT:    lsr w14, w14, #161917; NONEON-NOSVE-NEXT:    mul w10, w10, w181918; NONEON-NOSVE-NEXT:    lsr w13, w13, #161919; NONEON-NOSVE-NEXT:    strh w15, [sp, #78]1920; NONEON-NOSVE-NEXT:    mul w9, w9, w161921; NONEON-NOSVE-NEXT:    lsr w12, w12, #161922; NONEON-NOSVE-NEXT:    strh w14, [sp, #76]1923; NONEON-NOSVE-NEXT:    mul w8, w8, w171924; NONEON-NOSVE-NEXT:    lsr w11, w11, #161925; NONEON-NOSVE-NEXT:    strh w13, [sp, #74]1926; NONEON-NOSVE-NEXT:    lsr w10, w10, #161927; NONEON-NOSVE-NEXT:    strh w12, [sp, #72]1928; NONEON-NOSVE-NEXT:    lsr w9, w9, #161929; NONEON-NOSVE-NEXT:    strh w11, [sp, #70]1930; NONEON-NOSVE-NEXT:    lsr w8, w8, #161931; NONEON-NOSVE-NEXT:    strh w10, [sp, #68]1932; NONEON-NOSVE-NEXT:    strh w9, [sp, #66]1933; NONEON-NOSVE-NEXT:    strh w8, [sp, #64]1934; NONEON-NOSVE-NEXT:    ldr q0, [sp, #64]1935; NONEON-NOSVE-NEXT:    add sp, sp, #801936; NONEON-NOSVE-NEXT:    ret1937  %1 = zext <8 x i16> %op1 to <8 x i32>1938  %2 = zext <8 x i16> %op2 to <8 x i32>1939  %mul = mul <8 x i32> %1, %21940  %shr = lshr <8 x i32> %mul, splat (i32 16)1941  %res = trunc <8 x i32> %shr to <8 x i16>1942  ret <8 x i16> %res1943}1944 1945define void @umulh_v16i16(ptr %a, ptr %b) {1946; SVE-LABEL: umulh_v16i16:1947; SVE:       // %bb.0:1948; SVE-NEXT:    ldp q0, q3, [x1]1949; SVE-NEXT:    ptrue p0.h, vl81950; SVE-NEXT:    ldp q1, q2, [x0]1951; SVE-NEXT:    umulh z0.h, p0/m, z0.h, z1.h1952; SVE-NEXT:    umulh z2.h, p0/m, z2.h, z3.h1953; SVE-NEXT:    stp q0, q2, [x0]1954; SVE-NEXT:    ret1955;1956; SVE2-LABEL: umulh_v16i16:1957; SVE2:       // %bb.0:1958; SVE2-NEXT:    ldp q0, q3, [x1]1959; SVE2-NEXT:    ldp q1, q2, [x0]1960; SVE2-NEXT:    umulh z0.h, z1.h, z0.h1961; SVE2-NEXT:    umulh z1.h, z2.h, z3.h1962; SVE2-NEXT:    stp q0, q1, [x0]1963; SVE2-NEXT:    ret1964;1965; NONEON-NOSVE-LABEL: umulh_v16i16:1966; NONEON-NOSVE:       // %bb.0:1967; NONEON-NOSVE-NEXT:    sub sp, sp, #2401968; NONEON-NOSVE-NEXT:    stp x28, x27, [sp, #160] // 16-byte Folded Spill1969; NONEON-NOSVE-NEXT:    stp x26, x25, [sp, #176] // 16-byte Folded Spill1970; NONEON-NOSVE-NEXT:    stp x24, x23, [sp, #192] // 16-byte Folded Spill1971; NONEON-NOSVE-NEXT:    stp x22, x21, [sp, #208] // 16-byte Folded Spill1972; NONEON-NOSVE-NEXT:    stp x20, x19, [sp, #224] // 16-byte Folded Spill1973; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 2401974; NONEON-NOSVE-NEXT:    .cfi_offset w19, -81975; NONEON-NOSVE-NEXT:    .cfi_offset w20, -161976; NONEON-NOSVE-NEXT:    .cfi_offset w21, -241977; NONEON-NOSVE-NEXT:    .cfi_offset w22, -321978; NONEON-NOSVE-NEXT:    .cfi_offset w23, -401979; NONEON-NOSVE-NEXT:    .cfi_offset w24, -481980; NONEON-NOSVE-NEXT:    .cfi_offset w25, -561981; NONEON-NOSVE-NEXT:    .cfi_offset w26, -641982; NONEON-NOSVE-NEXT:    .cfi_offset w27, -721983; NONEON-NOSVE-NEXT:    .cfi_offset w28, -801984; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]1985; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]1986; NONEON-NOSVE-NEXT:    str q0, [sp]1987; NONEON-NOSVE-NEXT:    str q1, [sp, #32]1988; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]1989; NONEON-NOSVE-NEXT:    str q3, [sp, #16]1990; NONEON-NOSVE-NEXT:    str q2, [sp, #64]1991; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]1992; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #32]1993; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #56]1994; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #58]1995; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #60]1996; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #62]1997; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #48]1998; NONEON-NOSVE-NEXT:    ldrh w13, [sp, #50]1999; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]2000; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]2001; NONEON-NOSVE-NEXT:    ldrh w7, [sp, #96]2002; NONEON-NOSVE-NEXT:    ldrh w19, [sp, #98]2003; NONEON-NOSVE-NEXT:    ldrh w20, [sp, #100]2004; NONEON-NOSVE-NEXT:    ldrh w21, [sp, #102]2005; NONEON-NOSVE-NEXT:    ldrh w14, [sp, #52]2006; NONEON-NOSVE-NEXT:    ldrh w16, [sp, #54]2007; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]2008; NONEON-NOSVE-NEXT:    ldrh w18, [sp, #104]2009; NONEON-NOSVE-NEXT:    ldrh w2, [sp, #106]2010; NONEON-NOSVE-NEXT:    ldrh w4, [sp, #108]2011; NONEON-NOSVE-NEXT:    ldrh w5, [sp, #110]2012; NONEON-NOSVE-NEXT:    ldrh w15, [sp, #88]2013; NONEON-NOSVE-NEXT:    ldrh w17, [sp, #90]2014; NONEON-NOSVE-NEXT:    ldrh w1, [sp, #92]2015; NONEON-NOSVE-NEXT:    ldrh w3, [sp, #94]2016; NONEON-NOSVE-NEXT:    mul w8, w8, w152017; NONEON-NOSVE-NEXT:    ldrh w6, [sp, #80]2018; NONEON-NOSVE-NEXT:    ldrh w23, [sp, #82]2019; NONEON-NOSVE-NEXT:    mul w11, w11, w32020; NONEON-NOSVE-NEXT:    ldrh w25, [sp, #84]2021; NONEON-NOSVE-NEXT:    mul w13, w13, w232022; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #64]2023; NONEON-NOSVE-NEXT:    mul w14, w14, w252024; NONEON-NOSVE-NEXT:    lsr w8, w8, #162025; NONEON-NOSVE-NEXT:    mul w12, w12, w62026; NONEON-NOSVE-NEXT:    lsr w11, w11, #162027; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]2028; NONEON-NOSVE-NEXT:    mul w10, w10, w12029; NONEON-NOSVE-NEXT:    lsr w13, w13, #162030; NONEON-NOSVE-NEXT:    ldrh w22, [sp, #118]2031; NONEON-NOSVE-NEXT:    ldrh w24, [sp, #116]2032; NONEON-NOSVE-NEXT:    ldrh w26, [sp, #114]2033; NONEON-NOSVE-NEXT:    ldrh w27, [sp, #112]2034; NONEON-NOSVE-NEXT:    ldrh w28, [sp, #126]2035; NONEON-NOSVE-NEXT:    mul w9, w9, w172036; NONEON-NOSVE-NEXT:    mul w21, w21, w222037; NONEON-NOSVE-NEXT:    ldrh w22, [sp, #86]2038; NONEON-NOSVE-NEXT:    lsr w14, w14, #162039; NONEON-NOSVE-NEXT:    mul w20, w20, w242040; NONEON-NOSVE-NEXT:    ldrh w24, [sp, #120]2041; NONEON-NOSVE-NEXT:    lsr w12, w12, #162042; NONEON-NOSVE-NEXT:    mul w19, w19, w262043; NONEON-NOSVE-NEXT:    ldrh w26, [sp, #124]2044; NONEON-NOSVE-NEXT:    lsr w10, w10, #162045; NONEON-NOSVE-NEXT:    mul w7, w7, w272046; NONEON-NOSVE-NEXT:    ldrh w27, [sp, #122]2047; NONEON-NOSVE-NEXT:    lsr w21, w21, #162048; NONEON-NOSVE-NEXT:    mul w5, w5, w282049; NONEON-NOSVE-NEXT:    lsr w20, w20, #162050; NONEON-NOSVE-NEXT:    lsr w9, w9, #162051; NONEON-NOSVE-NEXT:    mul w4, w4, w262052; NONEON-NOSVE-NEXT:    lsr w19, w19, #162053; NONEON-NOSVE-NEXT:    strh w21, [sp, #158]2054; NONEON-NOSVE-NEXT:    mul w2, w2, w272055; NONEON-NOSVE-NEXT:    lsr w7, w7, #162056; NONEON-NOSVE-NEXT:    strh w20, [sp, #156]2057; NONEON-NOSVE-NEXT:    mul w18, w18, w242058; NONEON-NOSVE-NEXT:    lsr w5, w5, #162059; NONEON-NOSVE-NEXT:    strh w19, [sp, #154]2060; NONEON-NOSVE-NEXT:    mul w16, w16, w222061; NONEON-NOSVE-NEXT:    lsr w4, w4, #162062; NONEON-NOSVE-NEXT:    strh w7, [sp, #152]2063; NONEON-NOSVE-NEXT:    lsr w2, w2, #162064; NONEON-NOSVE-NEXT:    strh w5, [sp, #150]2065; NONEON-NOSVE-NEXT:    lsr w18, w18, #162066; NONEON-NOSVE-NEXT:    strh w4, [sp, #148]2067; NONEON-NOSVE-NEXT:    lsr w16, w16, #162068; NONEON-NOSVE-NEXT:    strh w2, [sp, #146]2069; NONEON-NOSVE-NEXT:    strh w18, [sp, #144]2070; NONEON-NOSVE-NEXT:    ldp x20, x19, [sp, #224] // 16-byte Folded Reload2071; NONEON-NOSVE-NEXT:    strh w16, [sp, #142]2072; NONEON-NOSVE-NEXT:    ldp x22, x21, [sp, #208] // 16-byte Folded Reload2073; NONEON-NOSVE-NEXT:    strh w14, [sp, #140]2074; NONEON-NOSVE-NEXT:    ldp x24, x23, [sp, #192] // 16-byte Folded Reload2075; NONEON-NOSVE-NEXT:    strh w13, [sp, #138]2076; NONEON-NOSVE-NEXT:    ldp x26, x25, [sp, #176] // 16-byte Folded Reload2077; NONEON-NOSVE-NEXT:    strh w12, [sp, #136]2078; NONEON-NOSVE-NEXT:    ldp x28, x27, [sp, #160] // 16-byte Folded Reload2079; NONEON-NOSVE-NEXT:    strh w11, [sp, #134]2080; NONEON-NOSVE-NEXT:    strh w10, [sp, #132]2081; NONEON-NOSVE-NEXT:    strh w9, [sp, #130]2082; NONEON-NOSVE-NEXT:    strh w8, [sp, #128]2083; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #128]2084; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]2085; NONEON-NOSVE-NEXT:    add sp, sp, #2402086; NONEON-NOSVE-NEXT:    ret2087  %op1 = load <16 x i16>, ptr %a2088  %op2 = load <16 x i16>, ptr %b2089  %1 = zext <16 x i16> %op1 to <16 x i32>2090  %2 = zext <16 x i16> %op2 to <16 x i32>2091  %mul = mul <16 x i32> %1, %22092  %shr = lshr <16 x i32> %mul, splat (i32 16)2093  %res = trunc <16 x i32> %shr to <16 x i16>2094  store <16 x i16> %res, ptr %a2095  ret void2096}2097 2098define <2 x i32> @umulh_v2i32(<2 x i32> %op1, <2 x i32> %op2) {2099; SVE-LABEL: umulh_v2i32:2100; SVE:       // %bb.0:2101; SVE-NEXT:    ptrue p0.s, vl22102; SVE-NEXT:    // kill: def $d0 killed $d0 def $z02103; SVE-NEXT:    // kill: def $d1 killed $d1 def $z12104; SVE-NEXT:    umulh z0.s, p0/m, z0.s, z1.s2105; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z02106; SVE-NEXT:    ret2107;2108; SVE2-LABEL: umulh_v2i32:2109; SVE2:       // %bb.0:2110; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z02111; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z12112; SVE2-NEXT:    umulh z0.s, z0.s, z1.s2113; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z02114; SVE2-NEXT:    ret2115;2116; NONEON-NOSVE-LABEL: umulh_v2i32:2117; NONEON-NOSVE:       // %bb.0:2118; NONEON-NOSVE-NEXT:    sub sp, sp, #322119; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 322120; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]2121; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #8]2122; NONEON-NOSVE-NEXT:    ldp w11, w10, [sp, #16]2123; NONEON-NOSVE-NEXT:    umull x9, w9, w102124; NONEON-NOSVE-NEXT:    umull x8, w8, w112125; NONEON-NOSVE-NEXT:    lsr x9, x9, #322126; NONEON-NOSVE-NEXT:    lsr x8, x8, #322127; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]2128; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]2129; NONEON-NOSVE-NEXT:    add sp, sp, #322130; NONEON-NOSVE-NEXT:    ret2131  %1 = zext <2 x i32> %op1 to <2 x i64>2132  %2 = zext <2 x i32> %op2 to <2 x i64>2133  %mul = mul <2 x i64> %1, %22134  %shr = lshr <2 x i64> %mul, splat (i64 32)2135  %res = trunc <2 x i64> %shr to <2 x i32>2136  ret <2 x i32> %res2137}2138 2139define <4 x i32> @umulh_v4i32(<4 x i32> %op1, <4 x i32> %op2) {2140; SVE-LABEL: umulh_v4i32:2141; SVE:       // %bb.0:2142; SVE-NEXT:    ptrue p0.s, vl42143; SVE-NEXT:    // kill: def $q0 killed $q0 def $z02144; SVE-NEXT:    // kill: def $q1 killed $q1 def $z12145; SVE-NEXT:    umulh z0.s, p0/m, z0.s, z1.s2146; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z02147; SVE-NEXT:    ret2148;2149; SVE2-LABEL: umulh_v4i32:2150; SVE2:       // %bb.0:2151; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z02152; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z12153; SVE2-NEXT:    umulh z0.s, z0.s, z1.s2154; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z02155; SVE2-NEXT:    ret2156;2157; NONEON-NOSVE-LABEL: umulh_v4i32:2158; NONEON-NOSVE:       // %bb.0:2159; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-80]!2160; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 802161; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]2162; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #32]2163; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]2164; NONEON-NOSVE-NEXT:    ldp w10, w11, [sp, #32]2165; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #40]2166; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]2167; NONEON-NOSVE-NEXT:    ldp w13, w12, [sp, #48]2168; NONEON-NOSVE-NEXT:    umull x11, w11, w122169; NONEON-NOSVE-NEXT:    ldp w14, w12, [sp, #56]2170; NONEON-NOSVE-NEXT:    umull x10, w10, w132171; NONEON-NOSVE-NEXT:    lsr x11, x11, #322172; NONEON-NOSVE-NEXT:    umull x9, w9, w122173; NONEON-NOSVE-NEXT:    umull x8, w8, w142174; NONEON-NOSVE-NEXT:    lsr x10, x10, #322175; NONEON-NOSVE-NEXT:    lsr x9, x9, #322176; NONEON-NOSVE-NEXT:    stp w10, w11, [sp, #72]2177; NONEON-NOSVE-NEXT:    lsr x8, x8, #322178; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #64]2179; NONEON-NOSVE-NEXT:    ldr q0, [sp, #64]2180; NONEON-NOSVE-NEXT:    add sp, sp, #802181; NONEON-NOSVE-NEXT:    ret2182  %1 = zext <4 x i32> %op1 to <4 x i64>2183  %2 = zext <4 x i32> %op2 to <4 x i64>2184  %mul = mul <4 x i64> %1, %22185  %shr = lshr <4 x i64> %mul, splat (i64 32)2186  %res = trunc <4 x i64> %shr to <4 x i32>2187  ret <4 x i32> %res2188}2189 2190define void @umulh_v8i32(ptr %a, ptr %b) {2191; SVE-LABEL: umulh_v8i32:2192; SVE:       // %bb.0:2193; SVE-NEXT:    ldp q0, q3, [x1]2194; SVE-NEXT:    ptrue p0.s, vl42195; SVE-NEXT:    ldp q1, q2, [x0]2196; SVE-NEXT:    umulh z0.s, p0/m, z0.s, z1.s2197; SVE-NEXT:    umulh z2.s, p0/m, z2.s, z3.s2198; SVE-NEXT:    stp q0, q2, [x0]2199; SVE-NEXT:    ret2200;2201; SVE2-LABEL: umulh_v8i32:2202; SVE2:       // %bb.0:2203; SVE2-NEXT:    ldp q0, q3, [x1]2204; SVE2-NEXT:    ldp q1, q2, [x0]2205; SVE2-NEXT:    umulh z0.s, z1.s, z0.s2206; SVE2-NEXT:    umulh z1.s, z2.s, z3.s2207; SVE2-NEXT:    stp q0, q1, [x0]2208; SVE2-NEXT:    ret2209;2210; NONEON-NOSVE-LABEL: umulh_v8i32:2211; NONEON-NOSVE:       // %bb.0:2212; NONEON-NOSVE-NEXT:    sub sp, sp, #1602213; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 1602214; NONEON-NOSVE-NEXT:    ldp q0, q1, [x0]2215; NONEON-NOSVE-NEXT:    ldp q3, q2, [x1]2216; NONEON-NOSVE-NEXT:    str q1, [sp, #32]2217; NONEON-NOSVE-NEXT:    stp q0, q3, [sp]2218; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp]2219; NONEON-NOSVE-NEXT:    str q2, [sp, #64]2220; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]2221; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #32]2222; NONEON-NOSVE-NEXT:    ldp w8, w9, [sp, #56]2223; NONEON-NOSVE-NEXT:    ldp w10, w11, [sp, #48]2224; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]2225; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #16]2226; NONEON-NOSVE-NEXT:    ldp w14, w15, [sp, #96]2227; NONEON-NOSVE-NEXT:    ldp w12, w13, [sp, #104]2228; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #80]2229; NONEON-NOSVE-NEXT:    ldp d1, d0, [sp, #64]2230; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]2231; NONEON-NOSVE-NEXT:    ldp w17, w16, [sp, #112]2232; NONEON-NOSVE-NEXT:    umull x15, w15, w162233; NONEON-NOSVE-NEXT:    ldp w18, w16, [sp, #120]2234; NONEON-NOSVE-NEXT:    umull x14, w14, w172235; NONEON-NOSVE-NEXT:    ldp w17, w1, [sp, #80]2236; NONEON-NOSVE-NEXT:    umull x13, w13, w162237; NONEON-NOSVE-NEXT:    lsr x15, x15, #322238; NONEON-NOSVE-NEXT:    umull x12, w12, w182239; NONEON-NOSVE-NEXT:    lsr x14, x14, #322240; NONEON-NOSVE-NEXT:    ldp w16, w18, [sp, #88]2241; NONEON-NOSVE-NEXT:    umull x11, w11, w12242; NONEON-NOSVE-NEXT:    lsr x13, x13, #322243; NONEON-NOSVE-NEXT:    stp w14, w15, [sp, #152]2244; NONEON-NOSVE-NEXT:    umull x10, w10, w172245; NONEON-NOSVE-NEXT:    lsr x12, x12, #322246; NONEON-NOSVE-NEXT:    umull x9, w9, w182247; NONEON-NOSVE-NEXT:    umull x8, w8, w162248; NONEON-NOSVE-NEXT:    lsr x11, x11, #322249; NONEON-NOSVE-NEXT:    stp w12, w13, [sp, #144]2250; NONEON-NOSVE-NEXT:    lsr x10, x10, #322251; NONEON-NOSVE-NEXT:    lsr x9, x9, #322252; NONEON-NOSVE-NEXT:    lsr x8, x8, #322253; NONEON-NOSVE-NEXT:    stp w10, w11, [sp, #136]2254; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #128]2255; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #128]2256; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]2257; NONEON-NOSVE-NEXT:    add sp, sp, #1602258; NONEON-NOSVE-NEXT:    ret2259  %op1 = load <8 x i32>, ptr %a2260  %op2 = load <8 x i32>, ptr %b2261  %1 = zext <8 x i32> %op1 to <8 x i64>2262  %2 = zext <8 x i32> %op2 to <8 x i64>2263  %mul = mul <8 x i64> %1, %22264  %shr = lshr <8 x i64> %mul, splat (i64 32)2265  %res = trunc <8 x i64> %shr to <8 x i32>2266  store <8 x i32> %res, ptr %a2267  ret void2268}2269 2270define <1 x i64> @umulh_v1i64(<1 x i64> %op1, <1 x i64> %op2) {2271; SVE-LABEL: umulh_v1i64:2272; SVE:       // %bb.0:2273; SVE-NEXT:    ptrue p0.d, vl12274; SVE-NEXT:    // kill: def $d0 killed $d0 def $z02275; SVE-NEXT:    // kill: def $d1 killed $d1 def $z12276; SVE-NEXT:    umulh z0.d, p0/m, z0.d, z1.d2277; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z02278; SVE-NEXT:    ret2279;2280; SVE2-LABEL: umulh_v1i64:2281; SVE2:       // %bb.0:2282; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z02283; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z12284; SVE2-NEXT:    umulh z0.d, z0.d, z1.d2285; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z02286; SVE2-NEXT:    ret2287;2288; NONEON-NOSVE-LABEL: umulh_v1i64:2289; NONEON-NOSVE:       // %bb.0:2290; NONEON-NOSVE-NEXT:    sub sp, sp, #162291; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 162292; NONEON-NOSVE-NEXT:    fmov x8, d02293; NONEON-NOSVE-NEXT:    fmov x9, d12294; NONEON-NOSVE-NEXT:    umulh x8, x8, x92295; NONEON-NOSVE-NEXT:    str x8, [sp, #8]2296; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]2297; NONEON-NOSVE-NEXT:    add sp, sp, #162298; NONEON-NOSVE-NEXT:    ret2299  %1 = zext <1 x i64> %op1 to <1 x i128>2300  %2 = zext <1 x i64> %op2 to <1 x i128>2301  %mul = mul <1 x i128> %1, %22302  %shr = lshr <1 x i128> %mul, splat (i128 64)2303  %res = trunc <1 x i128> %shr to <1 x i64>2304  ret <1 x i64> %res2305}2306 2307define <2 x i64> @umulh_v2i64(<2 x i64> %op1, <2 x i64> %op2) {2308; SVE-LABEL: umulh_v2i64:2309; SVE:       // %bb.0:2310; SVE-NEXT:    ptrue p0.d, vl22311; SVE-NEXT:    // kill: def $q0 killed $q0 def $z02312; SVE-NEXT:    // kill: def $q1 killed $q1 def $z12313; SVE-NEXT:    umulh z0.d, p0/m, z0.d, z1.d2314; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z02315; SVE-NEXT:    ret2316;2317; SVE2-LABEL: umulh_v2i64:2318; SVE2:       // %bb.0:2319; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z02320; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z12321; SVE2-NEXT:    umulh z0.d, z0.d, z1.d2322; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z02323; SVE2-NEXT:    ret2324;2325; NONEON-NOSVE-LABEL: umulh_v2i64:2326; NONEON-NOSVE:       // %bb.0:2327; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-64]!2328; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 642329; NONEON-NOSVE-NEXT:    ldp x9, x8, [sp]2330; NONEON-NOSVE-NEXT:    ldp x11, x10, [sp, #16]2331; NONEON-NOSVE-NEXT:    umulh x8, x8, x102332; NONEON-NOSVE-NEXT:    umulh x9, x9, x112333; NONEON-NOSVE-NEXT:    stp x9, x8, [sp, #32]2334; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #32]2335; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #48]2336; NONEON-NOSVE-NEXT:    ldr q0, [sp, #48]2337; NONEON-NOSVE-NEXT:    add sp, sp, #642338; NONEON-NOSVE-NEXT:    ret2339  %1 = zext <2 x i64> %op1 to <2 x i128>2340  %2 = zext <2 x i64> %op2 to <2 x i128>2341  %mul = mul <2 x i128> %1, %22342  %shr = lshr <2 x i128> %mul, splat (i128 64)2343  %res = trunc <2 x i128> %shr to <2 x i64>2344  ret <2 x i64> %res2345}2346 2347define void @umulh_v4i64(ptr %a, ptr %b) {2348; SVE-LABEL: umulh_v4i64:2349; SVE:       // %bb.0:2350; SVE-NEXT:    ldp q0, q3, [x1]2351; SVE-NEXT:    ptrue p0.d, vl22352; SVE-NEXT:    ldp q1, q2, [x0]2353; SVE-NEXT:    umulh z0.d, p0/m, z0.d, z1.d2354; SVE-NEXT:    umulh z2.d, p0/m, z2.d, z3.d2355; SVE-NEXT:    stp q0, q2, [x0]2356; SVE-NEXT:    ret2357;2358; SVE2-LABEL: umulh_v4i64:2359; SVE2:       // %bb.0:2360; SVE2-NEXT:    ldp q0, q3, [x1]2361; SVE2-NEXT:    ldp q1, q2, [x0]2362; SVE2-NEXT:    umulh z0.d, z1.d, z0.d2363; SVE2-NEXT:    umulh z1.d, z2.d, z3.d2364; SVE2-NEXT:    stp q0, q1, [x0]2365; SVE2-NEXT:    ret2366;2367; NONEON-NOSVE-LABEL: umulh_v4i64:2368; NONEON-NOSVE:       // %bb.0:2369; NONEON-NOSVE-NEXT:    sub sp, sp, #1282370; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 1282371; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]2372; NONEON-NOSVE-NEXT:    ldp q2, q3, [x1]2373; NONEON-NOSVE-NEXT:    stp q1, q2, [sp]2374; NONEON-NOSVE-NEXT:    ldp x11, x10, [sp]2375; NONEON-NOSVE-NEXT:    stp q0, q3, [sp, #32]2376; NONEON-NOSVE-NEXT:    ldp x13, x12, [sp, #16]2377; NONEON-NOSVE-NEXT:    ldp x9, x8, [sp, #32]2378; NONEON-NOSVE-NEXT:    umulh x10, x10, x122379; NONEON-NOSVE-NEXT:    ldp x14, x12, [sp, #48]2380; NONEON-NOSVE-NEXT:    umulh x11, x11, x132381; NONEON-NOSVE-NEXT:    umulh x8, x8, x122382; NONEON-NOSVE-NEXT:    umulh x9, x9, x142383; NONEON-NOSVE-NEXT:    stp x11, x10, [sp, #64]2384; NONEON-NOSVE-NEXT:    stp x9, x8, [sp, #80]2385; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #80]2386; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #112]2387; NONEON-NOSVE-NEXT:    ldp d0, d1, [sp, #64]2388; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #96]2389; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #96]2390; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]2391; NONEON-NOSVE-NEXT:    add sp, sp, #1282392; NONEON-NOSVE-NEXT:    ret2393  %op1 = load <4 x i64>, ptr %a2394  %op2 = load <4 x i64>, ptr %b2395  %1 = zext <4 x i64> %op1 to <4 x i128>2396  %2 = zext <4 x i64> %op2 to <4 x i128>2397  %mul = mul <4 x i128> %1, %22398  %shr = lshr <4 x i128> %mul, splat (i128 64)2399  %res = trunc <4 x i128> %shr to <4 x i64>2400  store <4 x i64> %res, ptr %a2401  ret void2402}2403