brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE23; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE414; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX15; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX26; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BW7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BWVL8; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=AVX,AVX512,AVX512DQ9; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq,+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512,AVX512DQVL10 11;12; vXi6413;14 15define i64 @test_v2i64(<2 x i64> %a0) {16; SSE-LABEL: test_v2i64:17; SSE:       # %bb.0:18; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]19; SSE-NEXT:    movdqa %xmm0, %xmm220; SSE-NEXT:    psrlq $32, %xmm221; SSE-NEXT:    pmuludq %xmm1, %xmm222; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]23; SSE-NEXT:    pmuludq %xmm0, %xmm324; SSE-NEXT:    paddq %xmm2, %xmm325; SSE-NEXT:    psllq $32, %xmm326; SSE-NEXT:    pmuludq %xmm1, %xmm027; SSE-NEXT:    paddq %xmm3, %xmm028; SSE-NEXT:    movq %xmm0, %rax29; SSE-NEXT:    retq30;31; AVX1OR2-LABEL: test_v2i64:32; AVX1OR2:       # %bb.0:33; AVX1OR2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]34; AVX1OR2-NEXT:    vpsrlq $32, %xmm0, %xmm235; AVX1OR2-NEXT:    vpmuludq %xmm1, %xmm2, %xmm236; AVX1OR2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]37; AVX1OR2-NEXT:    vpmuludq %xmm3, %xmm0, %xmm338; AVX1OR2-NEXT:    vpaddq %xmm2, %xmm3, %xmm239; AVX1OR2-NEXT:    vpsllq $32, %xmm2, %xmm240; AVX1OR2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm041; AVX1OR2-NEXT:    vpaddq %xmm2, %xmm0, %xmm042; AVX1OR2-NEXT:    vmovq %xmm0, %rax43; AVX1OR2-NEXT:    retq44;45; AVX512BW-LABEL: test_v2i64:46; AVX512BW:       # %bb.0:47; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]48; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm249; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm250; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]51; AVX512BW-NEXT:    vpmuludq %xmm3, %xmm0, %xmm352; AVX512BW-NEXT:    vpaddq %xmm2, %xmm3, %xmm253; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm254; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm055; AVX512BW-NEXT:    vpaddq %xmm2, %xmm0, %xmm056; AVX512BW-NEXT:    vmovq %xmm0, %rax57; AVX512BW-NEXT:    retq58;59; AVX512BWVL-LABEL: test_v2i64:60; AVX512BWVL:       # %bb.0:61; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]62; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm263; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm264; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]65; AVX512BWVL-NEXT:    vpmuludq %xmm3, %xmm0, %xmm366; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm3, %xmm267; AVX512BWVL-NEXT:    vpsllq $32, %xmm2, %xmm268; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm069; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm0, %xmm070; AVX512BWVL-NEXT:    vmovq %xmm0, %rax71; AVX512BWVL-NEXT:    retq72;73; AVX512DQ-LABEL: test_v2i64:74; AVX512DQ:       # %bb.0:75; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm076; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]77; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm078; AVX512DQ-NEXT:    vmovq %xmm0, %rax79; AVX512DQ-NEXT:    vzeroupper80; AVX512DQ-NEXT:    retq81;82; AVX512DQVL-LABEL: test_v2i64:83; AVX512DQVL:       # %bb.0:84; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]85; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm086; AVX512DQVL-NEXT:    vmovq %xmm0, %rax87; AVX512DQVL-NEXT:    retq88  %1 = call i64 @llvm.vector.reduce.mul.v2i64(<2 x i64> %a0)89  ret i64 %190}91 92define i64 @test_v4i64(<4 x i64> %a0) {93; SSE-LABEL: test_v4i64:94; SSE:       # %bb.0:95; SSE-NEXT:    movdqa %xmm0, %xmm296; SSE-NEXT:    psrlq $32, %xmm297; SSE-NEXT:    pmuludq %xmm1, %xmm298; SSE-NEXT:    movdqa %xmm1, %xmm399; SSE-NEXT:    psrlq $32, %xmm3100; SSE-NEXT:    pmuludq %xmm0, %xmm3101; SSE-NEXT:    paddq %xmm2, %xmm3102; SSE-NEXT:    psllq $32, %xmm3103; SSE-NEXT:    pmuludq %xmm1, %xmm0104; SSE-NEXT:    paddq %xmm3, %xmm0105; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]106; SSE-NEXT:    movdqa %xmm0, %xmm2107; SSE-NEXT:    psrlq $32, %xmm2108; SSE-NEXT:    pmuludq %xmm1, %xmm2109; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]110; SSE-NEXT:    pmuludq %xmm0, %xmm3111; SSE-NEXT:    paddq %xmm2, %xmm3112; SSE-NEXT:    psllq $32, %xmm3113; SSE-NEXT:    pmuludq %xmm1, %xmm0114; SSE-NEXT:    paddq %xmm3, %xmm0115; SSE-NEXT:    movq %xmm0, %rax116; SSE-NEXT:    retq117;118; AVX1-LABEL: test_v4i64:119; AVX1:       # %bb.0:120; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1121; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2122; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2123; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3124; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3125; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2126; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2127; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0128; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0129; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]130; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2131; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2132; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]133; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3134; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2135; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2136; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0137; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0138; AVX1-NEXT:    vmovq %xmm0, %rax139; AVX1-NEXT:    vzeroupper140; AVX1-NEXT:    retq141;142; AVX2-LABEL: test_v4i64:143; AVX2:       # %bb.0:144; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1145; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2146; AVX2-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2147; AVX2-NEXT:    vpsrlq $32, %xmm1, %xmm3148; AVX2-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3149; AVX2-NEXT:    vpaddq %xmm2, %xmm3, %xmm2150; AVX2-NEXT:    vpsllq $32, %xmm2, %xmm2151; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0152; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0153; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]154; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2155; AVX2-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2156; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]157; AVX2-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3158; AVX2-NEXT:    vpaddq %xmm2, %xmm3, %xmm2159; AVX2-NEXT:    vpsllq $32, %xmm2, %xmm2160; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0161; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0162; AVX2-NEXT:    vmovq %xmm0, %rax163; AVX2-NEXT:    vzeroupper164; AVX2-NEXT:    retq165;166; AVX512BW-LABEL: test_v4i64:167; AVX512BW:       # %bb.0:168; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1169; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2170; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2171; AVX512BW-NEXT:    vpsrlq $32, %xmm1, %xmm3172; AVX512BW-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3173; AVX512BW-NEXT:    vpaddq %xmm2, %xmm3, %xmm2174; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm2175; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0176; AVX512BW-NEXT:    vpaddq %xmm2, %xmm0, %xmm0177; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]178; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2179; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2180; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]181; AVX512BW-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3182; AVX512BW-NEXT:    vpaddq %xmm2, %xmm3, %xmm2183; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm2184; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0185; AVX512BW-NEXT:    vpaddq %xmm2, %xmm0, %xmm0186; AVX512BW-NEXT:    vmovq %xmm0, %rax187; AVX512BW-NEXT:    vzeroupper188; AVX512BW-NEXT:    retq189;190; AVX512BWVL-LABEL: test_v4i64:191; AVX512BWVL:       # %bb.0:192; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1193; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm2194; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2195; AVX512BWVL-NEXT:    vpsrlq $32, %xmm1, %xmm3196; AVX512BWVL-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3197; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm3, %xmm2198; AVX512BWVL-NEXT:    vpsllq $32, %xmm2, %xmm2199; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0200; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0201; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]202; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm2203; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2204; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]205; AVX512BWVL-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3206; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm3, %xmm2207; AVX512BWVL-NEXT:    vpsllq $32, %xmm2, %xmm2208; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0209; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0210; AVX512BWVL-NEXT:    vmovq %xmm0, %rax211; AVX512BWVL-NEXT:    vzeroupper212; AVX512BWVL-NEXT:    retq213;214; AVX512DQ-LABEL: test_v4i64:215; AVX512DQ:       # %bb.0:216; AVX512DQ-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0217; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1218; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0219; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]220; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0221; AVX512DQ-NEXT:    vmovq %xmm0, %rax222; AVX512DQ-NEXT:    vzeroupper223; AVX512DQ-NEXT:    retq224;225; AVX512DQVL-LABEL: test_v4i64:226; AVX512DQVL:       # %bb.0:227; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1228; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0229; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]230; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0231; AVX512DQVL-NEXT:    vmovq %xmm0, %rax232; AVX512DQVL-NEXT:    vzeroupper233; AVX512DQVL-NEXT:    retq234  %1 = call i64 @llvm.vector.reduce.mul.v4i64(<4 x i64> %a0)235  ret i64 %1236}237 238define i64 @test_v8i64(<8 x i64> %a0) {239; SSE-LABEL: test_v8i64:240; SSE:       # %bb.0:241; SSE-NEXT:    movdqa %xmm1, %xmm4242; SSE-NEXT:    psrlq $32, %xmm4243; SSE-NEXT:    pmuludq %xmm3, %xmm4244; SSE-NEXT:    movdqa %xmm3, %xmm5245; SSE-NEXT:    psrlq $32, %xmm5246; SSE-NEXT:    pmuludq %xmm1, %xmm5247; SSE-NEXT:    paddq %xmm4, %xmm5248; SSE-NEXT:    psllq $32, %xmm5249; SSE-NEXT:    pmuludq %xmm3, %xmm1250; SSE-NEXT:    paddq %xmm5, %xmm1251; SSE-NEXT:    movdqa %xmm0, %xmm3252; SSE-NEXT:    psrlq $32, %xmm3253; SSE-NEXT:    pmuludq %xmm2, %xmm3254; SSE-NEXT:    movdqa %xmm2, %xmm4255; SSE-NEXT:    psrlq $32, %xmm4256; SSE-NEXT:    pmuludq %xmm0, %xmm4257; SSE-NEXT:    paddq %xmm3, %xmm4258; SSE-NEXT:    psllq $32, %xmm4259; SSE-NEXT:    pmuludq %xmm2, %xmm0260; SSE-NEXT:    paddq %xmm4, %xmm0261; SSE-NEXT:    movdqa %xmm0, %xmm2262; SSE-NEXT:    psrlq $32, %xmm2263; SSE-NEXT:    pmuludq %xmm1, %xmm2264; SSE-NEXT:    movdqa %xmm1, %xmm3265; SSE-NEXT:    psrlq $32, %xmm3266; SSE-NEXT:    pmuludq %xmm0, %xmm3267; SSE-NEXT:    paddq %xmm2, %xmm3268; SSE-NEXT:    psllq $32, %xmm3269; SSE-NEXT:    pmuludq %xmm1, %xmm0270; SSE-NEXT:    paddq %xmm3, %xmm0271; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]272; SSE-NEXT:    movdqa %xmm0, %xmm2273; SSE-NEXT:    psrlq $32, %xmm2274; SSE-NEXT:    pmuludq %xmm1, %xmm2275; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]276; SSE-NEXT:    pmuludq %xmm0, %xmm3277; SSE-NEXT:    paddq %xmm2, %xmm3278; SSE-NEXT:    psllq $32, %xmm3279; SSE-NEXT:    pmuludq %xmm1, %xmm0280; SSE-NEXT:    paddq %xmm3, %xmm0281; SSE-NEXT:    movq %xmm0, %rax282; SSE-NEXT:    retq283;284; AVX1-LABEL: test_v8i64:285; AVX1:       # %bb.0:286; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2287; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3288; AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm4289; AVX1-NEXT:    vpmuludq %xmm2, %xmm4, %xmm4290; AVX1-NEXT:    vpsrlq $32, %xmm2, %xmm5291; AVX1-NEXT:    vpmuludq %xmm5, %xmm3, %xmm5292; AVX1-NEXT:    vpaddq %xmm4, %xmm5, %xmm4293; AVX1-NEXT:    vpsllq $32, %xmm4, %xmm4294; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2295; AVX1-NEXT:    vpaddq %xmm4, %xmm2, %xmm2296; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm3297; AVX1-NEXT:    vpmuludq %xmm1, %xmm3, %xmm3298; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm4299; AVX1-NEXT:    vpmuludq %xmm4, %xmm0, %xmm4300; AVX1-NEXT:    vpaddq %xmm3, %xmm4, %xmm3301; AVX1-NEXT:    vpsllq $32, %xmm3, %xmm3302; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0303; AVX1-NEXT:    vpaddq %xmm3, %xmm0, %xmm0304; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm1305; AVX1-NEXT:    vpmuludq %xmm2, %xmm1, %xmm1306; AVX1-NEXT:    vpsrlq $32, %xmm2, %xmm3307; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3308; AVX1-NEXT:    vpaddq %xmm1, %xmm3, %xmm1309; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm1310; AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm0311; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0312; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]313; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2314; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2315; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]316; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3317; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2318; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2319; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0320; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0321; AVX1-NEXT:    vmovq %xmm0, %rax322; AVX1-NEXT:    vzeroupper323; AVX1-NEXT:    retq324;325; AVX2-LABEL: test_v8i64:326; AVX2:       # %bb.0:327; AVX2-NEXT:    vpsrlq $32, %ymm0, %ymm2328; AVX2-NEXT:    vpmuludq %ymm1, %ymm2, %ymm2329; AVX2-NEXT:    vpsrlq $32, %ymm1, %ymm3330; AVX2-NEXT:    vpmuludq %ymm3, %ymm0, %ymm3331; AVX2-NEXT:    vpaddq %ymm2, %ymm3, %ymm2332; AVX2-NEXT:    vpsllq $32, %ymm2, %ymm2333; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0334; AVX2-NEXT:    vpaddq %ymm2, %ymm0, %ymm0335; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1336; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2337; AVX2-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2338; AVX2-NEXT:    vpsrlq $32, %xmm1, %xmm3339; AVX2-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3340; AVX2-NEXT:    vpaddq %xmm2, %xmm3, %xmm2341; AVX2-NEXT:    vpsllq $32, %xmm2, %xmm2342; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0343; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0344; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]345; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2346; AVX2-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2347; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]348; AVX2-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3349; AVX2-NEXT:    vpaddq %xmm2, %xmm3, %xmm2350; AVX2-NEXT:    vpsllq $32, %xmm2, %xmm2351; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0352; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0353; AVX2-NEXT:    vmovq %xmm0, %rax354; AVX2-NEXT:    vzeroupper355; AVX2-NEXT:    retq356;357; AVX512BW-LABEL: test_v8i64:358; AVX512BW:       # %bb.0:359; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1360; AVX512BW-NEXT:    vpsrlq $32, %ymm0, %ymm2361; AVX512BW-NEXT:    vpmuludq %ymm1, %ymm2, %ymm2362; AVX512BW-NEXT:    vpsrlq $32, %ymm1, %ymm3363; AVX512BW-NEXT:    vpmuludq %ymm3, %ymm0, %ymm3364; AVX512BW-NEXT:    vpaddq %ymm2, %ymm3, %ymm2365; AVX512BW-NEXT:    vpsllq $32, %ymm2, %ymm2366; AVX512BW-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0367; AVX512BW-NEXT:    vpaddq %ymm2, %ymm0, %ymm0368; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1369; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2370; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2371; AVX512BW-NEXT:    vpsrlq $32, %xmm1, %xmm3372; AVX512BW-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3373; AVX512BW-NEXT:    vpaddq %xmm2, %xmm3, %xmm2374; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm2375; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0376; AVX512BW-NEXT:    vpaddq %xmm2, %xmm0, %xmm0377; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]378; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2379; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2380; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]381; AVX512BW-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3382; AVX512BW-NEXT:    vpaddq %xmm2, %xmm3, %xmm2383; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm2384; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0385; AVX512BW-NEXT:    vpaddq %xmm2, %xmm0, %xmm0386; AVX512BW-NEXT:    vmovq %xmm0, %rax387; AVX512BW-NEXT:    vzeroupper388; AVX512BW-NEXT:    retq389;390; AVX512BWVL-LABEL: test_v8i64:391; AVX512BWVL:       # %bb.0:392; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1393; AVX512BWVL-NEXT:    vpsrlq $32, %ymm0, %ymm2394; AVX512BWVL-NEXT:    vpmuludq %ymm1, %ymm2, %ymm2395; AVX512BWVL-NEXT:    vpsrlq $32, %ymm1, %ymm3396; AVX512BWVL-NEXT:    vpmuludq %ymm3, %ymm0, %ymm3397; AVX512BWVL-NEXT:    vpaddq %ymm2, %ymm3, %ymm2398; AVX512BWVL-NEXT:    vpsllq $32, %ymm2, %ymm2399; AVX512BWVL-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0400; AVX512BWVL-NEXT:    vpaddq %ymm2, %ymm0, %ymm0401; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1402; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm2403; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2404; AVX512BWVL-NEXT:    vpsrlq $32, %xmm1, %xmm3405; AVX512BWVL-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3406; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm3, %xmm2407; AVX512BWVL-NEXT:    vpsllq $32, %xmm2, %xmm2408; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0409; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0410; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]411; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm2412; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2413; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]414; AVX512BWVL-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3415; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm3, %xmm2416; AVX512BWVL-NEXT:    vpsllq $32, %xmm2, %xmm2417; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0418; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0419; AVX512BWVL-NEXT:    vmovq %xmm0, %rax420; AVX512BWVL-NEXT:    vzeroupper421; AVX512BWVL-NEXT:    retq422;423; AVX512DQ-LABEL: test_v8i64:424; AVX512DQ:       # %bb.0:425; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1426; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0427; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1428; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0429; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]430; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0431; AVX512DQ-NEXT:    vmovq %xmm0, %rax432; AVX512DQ-NEXT:    vzeroupper433; AVX512DQ-NEXT:    retq434;435; AVX512DQVL-LABEL: test_v8i64:436; AVX512DQVL:       # %bb.0:437; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1438; AVX512DQVL-NEXT:    vpmullq %zmm1, %zmm0, %zmm0439; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1440; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0441; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]442; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0443; AVX512DQVL-NEXT:    vmovq %xmm0, %rax444; AVX512DQVL-NEXT:    vzeroupper445; AVX512DQVL-NEXT:    retq446  %1 = call i64 @llvm.vector.reduce.mul.v8i64(<8 x i64> %a0)447  ret i64 %1448}449 450define i64 @test_v16i64(<16 x i64> %a0) {451; SSE-LABEL: test_v16i64:452; SSE:       # %bb.0:453; SSE-NEXT:    movdqa %xmm2, %xmm8454; SSE-NEXT:    psrlq $32, %xmm8455; SSE-NEXT:    pmuludq %xmm6, %xmm8456; SSE-NEXT:    movdqa %xmm6, %xmm9457; SSE-NEXT:    psrlq $32, %xmm9458; SSE-NEXT:    pmuludq %xmm2, %xmm9459; SSE-NEXT:    paddq %xmm8, %xmm9460; SSE-NEXT:    psllq $32, %xmm9461; SSE-NEXT:    pmuludq %xmm6, %xmm2462; SSE-NEXT:    paddq %xmm9, %xmm2463; SSE-NEXT:    movdqa %xmm0, %xmm6464; SSE-NEXT:    psrlq $32, %xmm6465; SSE-NEXT:    pmuludq %xmm4, %xmm6466; SSE-NEXT:    movdqa %xmm4, %xmm8467; SSE-NEXT:    psrlq $32, %xmm8468; SSE-NEXT:    pmuludq %xmm0, %xmm8469; SSE-NEXT:    paddq %xmm6, %xmm8470; SSE-NEXT:    psllq $32, %xmm8471; SSE-NEXT:    pmuludq %xmm4, %xmm0472; SSE-NEXT:    paddq %xmm8, %xmm0473; SSE-NEXT:    movdqa %xmm3, %xmm4474; SSE-NEXT:    psrlq $32, %xmm4475; SSE-NEXT:    pmuludq %xmm7, %xmm4476; SSE-NEXT:    movdqa %xmm7, %xmm6477; SSE-NEXT:    psrlq $32, %xmm6478; SSE-NEXT:    pmuludq %xmm3, %xmm6479; SSE-NEXT:    paddq %xmm4, %xmm6480; SSE-NEXT:    psllq $32, %xmm6481; SSE-NEXT:    pmuludq %xmm7, %xmm3482; SSE-NEXT:    paddq %xmm6, %xmm3483; SSE-NEXT:    movdqa %xmm1, %xmm4484; SSE-NEXT:    psrlq $32, %xmm4485; SSE-NEXT:    pmuludq %xmm5, %xmm4486; SSE-NEXT:    movdqa %xmm5, %xmm6487; SSE-NEXT:    psrlq $32, %xmm6488; SSE-NEXT:    pmuludq %xmm1, %xmm6489; SSE-NEXT:    paddq %xmm4, %xmm6490; SSE-NEXT:    psllq $32, %xmm6491; SSE-NEXT:    pmuludq %xmm5, %xmm1492; SSE-NEXT:    paddq %xmm6, %xmm1493; SSE-NEXT:    movdqa %xmm1, %xmm4494; SSE-NEXT:    psrlq $32, %xmm4495; SSE-NEXT:    pmuludq %xmm3, %xmm4496; SSE-NEXT:    movdqa %xmm3, %xmm5497; SSE-NEXT:    psrlq $32, %xmm5498; SSE-NEXT:    pmuludq %xmm1, %xmm5499; SSE-NEXT:    paddq %xmm4, %xmm5500; SSE-NEXT:    psllq $32, %xmm5501; SSE-NEXT:    pmuludq %xmm3, %xmm1502; SSE-NEXT:    paddq %xmm5, %xmm1503; SSE-NEXT:    movdqa %xmm0, %xmm3504; SSE-NEXT:    psrlq $32, %xmm3505; SSE-NEXT:    pmuludq %xmm2, %xmm3506; SSE-NEXT:    movdqa %xmm2, %xmm4507; SSE-NEXT:    psrlq $32, %xmm4508; SSE-NEXT:    pmuludq %xmm0, %xmm4509; SSE-NEXT:    paddq %xmm3, %xmm4510; SSE-NEXT:    psllq $32, %xmm4511; SSE-NEXT:    pmuludq %xmm2, %xmm0512; SSE-NEXT:    paddq %xmm4, %xmm0513; SSE-NEXT:    movdqa %xmm0, %xmm2514; SSE-NEXT:    psrlq $32, %xmm2515; SSE-NEXT:    pmuludq %xmm1, %xmm2516; SSE-NEXT:    movdqa %xmm1, %xmm3517; SSE-NEXT:    psrlq $32, %xmm3518; SSE-NEXT:    pmuludq %xmm0, %xmm3519; SSE-NEXT:    paddq %xmm2, %xmm3520; SSE-NEXT:    psllq $32, %xmm3521; SSE-NEXT:    pmuludq %xmm1, %xmm0522; SSE-NEXT:    paddq %xmm3, %xmm0523; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]524; SSE-NEXT:    movdqa %xmm0, %xmm2525; SSE-NEXT:    psrlq $32, %xmm2526; SSE-NEXT:    pmuludq %xmm1, %xmm2527; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]528; SSE-NEXT:    pmuludq %xmm0, %xmm3529; SSE-NEXT:    paddq %xmm2, %xmm3530; SSE-NEXT:    psllq $32, %xmm3531; SSE-NEXT:    pmuludq %xmm1, %xmm0532; SSE-NEXT:    paddq %xmm3, %xmm0533; SSE-NEXT:    movq %xmm0, %rax534; SSE-NEXT:    retq535;536; AVX1-LABEL: test_v16i64:537; AVX1:       # %bb.0:538; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm4539; AVX1-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4540; AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm5541; AVX1-NEXT:    vpmuludq %xmm5, %xmm1, %xmm5542; AVX1-NEXT:    vpaddq %xmm4, %xmm5, %xmm4543; AVX1-NEXT:    vpsllq $32, %xmm4, %xmm4544; AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm5545; AVX1-NEXT:    vpaddq %xmm4, %xmm5, %xmm4546; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm5547; AVX1-NEXT:    vpmuludq %xmm2, %xmm5, %xmm5548; AVX1-NEXT:    vpsrlq $32, %xmm2, %xmm6549; AVX1-NEXT:    vpmuludq %xmm6, %xmm0, %xmm6550; AVX1-NEXT:    vpaddq %xmm5, %xmm6, %xmm5551; AVX1-NEXT:    vpsllq $32, %xmm5, %xmm5552; AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm6553; AVX1-NEXT:    vpaddq %xmm5, %xmm6, %xmm5554; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm3555; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1556; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm6557; AVX1-NEXT:    vpmuludq %xmm3, %xmm6, %xmm6558; AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm7559; AVX1-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7560; AVX1-NEXT:    vpaddq %xmm6, %xmm7, %xmm6561; AVX1-NEXT:    vpsllq $32, %xmm6, %xmm6562; AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1563; AVX1-NEXT:    vpaddq %xmm6, %xmm1, %xmm1564; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm2565; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0566; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm3567; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm3568; AVX1-NEXT:    vpsrlq $32, %xmm2, %xmm6569; AVX1-NEXT:    vpmuludq %xmm6, %xmm0, %xmm6570; AVX1-NEXT:    vpaddq %xmm3, %xmm6, %xmm3571; AVX1-NEXT:    vpsllq $32, %xmm3, %xmm3572; AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm0573; AVX1-NEXT:    vpaddq %xmm3, %xmm0, %xmm0574; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2575; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2576; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3577; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3578; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2579; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2580; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0581; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0582; AVX1-NEXT:    vpsrlq $32, %xmm5, %xmm1583; AVX1-NEXT:    vpmuludq %xmm4, %xmm1, %xmm1584; AVX1-NEXT:    vpsrlq $32, %xmm4, %xmm2585; AVX1-NEXT:    vpmuludq %xmm2, %xmm5, %xmm2586; AVX1-NEXT:    vpaddq %xmm1, %xmm2, %xmm1587; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm1588; AVX1-NEXT:    vpmuludq %xmm4, %xmm5, %xmm2589; AVX1-NEXT:    vpaddq %xmm1, %xmm2, %xmm1590; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm2591; AVX1-NEXT:    vpmuludq %xmm0, %xmm2, %xmm2592; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm3593; AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm3594; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2595; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2596; AVX1-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0597; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0598; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]599; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2600; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2601; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]602; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3603; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2604; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2605; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0606; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0607; AVX1-NEXT:    vmovq %xmm0, %rax608; AVX1-NEXT:    vzeroupper609; AVX1-NEXT:    retq610;611; AVX2-LABEL: test_v16i64:612; AVX2:       # %bb.0:613; AVX2-NEXT:    vpsrlq $32, %ymm1, %ymm4614; AVX2-NEXT:    vpmuludq %ymm3, %ymm4, %ymm4615; AVX2-NEXT:    vpsrlq $32, %ymm3, %ymm5616; AVX2-NEXT:    vpmuludq %ymm5, %ymm1, %ymm5617; AVX2-NEXT:    vpaddq %ymm4, %ymm5, %ymm4618; AVX2-NEXT:    vpsllq $32, %ymm4, %ymm4619; AVX2-NEXT:    vpmuludq %ymm3, %ymm1, %ymm1620; AVX2-NEXT:    vpaddq %ymm4, %ymm1, %ymm1621; AVX2-NEXT:    vpsrlq $32, %ymm0, %ymm3622; AVX2-NEXT:    vpmuludq %ymm2, %ymm3, %ymm3623; AVX2-NEXT:    vpsrlq $32, %ymm2, %ymm4624; AVX2-NEXT:    vpmuludq %ymm4, %ymm0, %ymm4625; AVX2-NEXT:    vpaddq %ymm3, %ymm4, %ymm3626; AVX2-NEXT:    vpsllq $32, %ymm3, %ymm3627; AVX2-NEXT:    vpmuludq %ymm2, %ymm0, %ymm0628; AVX2-NEXT:    vpaddq %ymm3, %ymm0, %ymm0629; AVX2-NEXT:    vpsrlq $32, %ymm0, %ymm2630; AVX2-NEXT:    vpmuludq %ymm1, %ymm2, %ymm2631; AVX2-NEXT:    vpsrlq $32, %ymm1, %ymm3632; AVX2-NEXT:    vpmuludq %ymm3, %ymm0, %ymm3633; AVX2-NEXT:    vpaddq %ymm2, %ymm3, %ymm2634; AVX2-NEXT:    vpsllq $32, %ymm2, %ymm2635; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0636; AVX2-NEXT:    vpaddq %ymm2, %ymm0, %ymm0637; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1638; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2639; AVX2-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2640; AVX2-NEXT:    vpsrlq $32, %xmm1, %xmm3641; AVX2-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3642; AVX2-NEXT:    vpaddq %xmm2, %xmm3, %xmm2643; AVX2-NEXT:    vpsllq $32, %xmm2, %xmm2644; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0645; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0646; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]647; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2648; AVX2-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2649; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]650; AVX2-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3651; AVX2-NEXT:    vpaddq %xmm2, %xmm3, %xmm2652; AVX2-NEXT:    vpsllq $32, %xmm2, %xmm2653; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0654; AVX2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0655; AVX2-NEXT:    vmovq %xmm0, %rax656; AVX2-NEXT:    vzeroupper657; AVX2-NEXT:    retq658;659; AVX512BW-LABEL: test_v16i64:660; AVX512BW:       # %bb.0:661; AVX512BW-NEXT:    vpsrlq $32, %zmm0, %zmm2662; AVX512BW-NEXT:    vpmuludq %zmm1, %zmm2, %zmm2663; AVX512BW-NEXT:    vpsrlq $32, %zmm1, %zmm3664; AVX512BW-NEXT:    vpmuludq %zmm3, %zmm0, %zmm3665; AVX512BW-NEXT:    vpaddq %zmm2, %zmm3, %zmm2666; AVX512BW-NEXT:    vpsllq $32, %zmm2, %zmm2667; AVX512BW-NEXT:    vpmuludq %zmm1, %zmm0, %zmm0668; AVX512BW-NEXT:    vpaddq %zmm2, %zmm0, %zmm0669; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1670; AVX512BW-NEXT:    vpsrlq $32, %ymm0, %ymm2671; AVX512BW-NEXT:    vpmuludq %ymm1, %ymm2, %ymm2672; AVX512BW-NEXT:    vpsrlq $32, %ymm1, %ymm3673; AVX512BW-NEXT:    vpmuludq %ymm3, %ymm0, %ymm3674; AVX512BW-NEXT:    vpaddq %ymm2, %ymm3, %ymm2675; AVX512BW-NEXT:    vpsllq $32, %ymm2, %ymm2676; AVX512BW-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0677; AVX512BW-NEXT:    vpaddq %ymm2, %ymm0, %ymm0678; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1679; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2680; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2681; AVX512BW-NEXT:    vpsrlq $32, %xmm1, %xmm3682; AVX512BW-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3683; AVX512BW-NEXT:    vpaddq %xmm2, %xmm3, %xmm2684; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm2685; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0686; AVX512BW-NEXT:    vpaddq %xmm2, %xmm0, %xmm0687; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]688; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2689; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2690; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]691; AVX512BW-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3692; AVX512BW-NEXT:    vpaddq %xmm2, %xmm3, %xmm2693; AVX512BW-NEXT:    vpsllq $32, %xmm2, %xmm2694; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0695; AVX512BW-NEXT:    vpaddq %xmm2, %xmm0, %xmm0696; AVX512BW-NEXT:    vmovq %xmm0, %rax697; AVX512BW-NEXT:    vzeroupper698; AVX512BW-NEXT:    retq699;700; AVX512BWVL-LABEL: test_v16i64:701; AVX512BWVL:       # %bb.0:702; AVX512BWVL-NEXT:    vpsrlq $32, %zmm0, %zmm2703; AVX512BWVL-NEXT:    vpmuludq %zmm1, %zmm2, %zmm2704; AVX512BWVL-NEXT:    vpsrlq $32, %zmm1, %zmm3705; AVX512BWVL-NEXT:    vpmuludq %zmm3, %zmm0, %zmm3706; AVX512BWVL-NEXT:    vpaddq %zmm2, %zmm3, %zmm2707; AVX512BWVL-NEXT:    vpsllq $32, %zmm2, %zmm2708; AVX512BWVL-NEXT:    vpmuludq %zmm1, %zmm0, %zmm0709; AVX512BWVL-NEXT:    vpaddq %zmm2, %zmm0, %zmm0710; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1711; AVX512BWVL-NEXT:    vpsrlq $32, %ymm0, %ymm2712; AVX512BWVL-NEXT:    vpmuludq %ymm1, %ymm2, %ymm2713; AVX512BWVL-NEXT:    vpsrlq $32, %ymm1, %ymm3714; AVX512BWVL-NEXT:    vpmuludq %ymm3, %ymm0, %ymm3715; AVX512BWVL-NEXT:    vpaddq %ymm2, %ymm3, %ymm2716; AVX512BWVL-NEXT:    vpsllq $32, %ymm2, %ymm2717; AVX512BWVL-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0718; AVX512BWVL-NEXT:    vpaddq %ymm2, %ymm0, %ymm0719; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1720; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm2721; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2722; AVX512BWVL-NEXT:    vpsrlq $32, %xmm1, %xmm3723; AVX512BWVL-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3724; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm3, %xmm2725; AVX512BWVL-NEXT:    vpsllq $32, %xmm2, %xmm2726; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0727; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0728; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]729; AVX512BWVL-NEXT:    vpsrlq $32, %xmm0, %xmm2730; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2731; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]732; AVX512BWVL-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3733; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm3, %xmm2734; AVX512BWVL-NEXT:    vpsllq $32, %xmm2, %xmm2735; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0736; AVX512BWVL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0737; AVX512BWVL-NEXT:    vmovq %xmm0, %rax738; AVX512BWVL-NEXT:    vzeroupper739; AVX512BWVL-NEXT:    retq740;741; AVX512DQ-LABEL: test_v16i64:742; AVX512DQ:       # %bb.0:743; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0744; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1745; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0746; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1747; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0748; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]749; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0750; AVX512DQ-NEXT:    vmovq %xmm0, %rax751; AVX512DQ-NEXT:    vzeroupper752; AVX512DQ-NEXT:    retq753;754; AVX512DQVL-LABEL: test_v16i64:755; AVX512DQVL:       # %bb.0:756; AVX512DQVL-NEXT:    vpmullq %zmm1, %zmm0, %zmm0757; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1758; AVX512DQVL-NEXT:    vpmullq %zmm1, %zmm0, %zmm0759; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1760; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0761; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]762; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0763; AVX512DQVL-NEXT:    vmovq %xmm0, %rax764; AVX512DQVL-NEXT:    vzeroupper765; AVX512DQVL-NEXT:    retq766  %1 = call i64 @llvm.vector.reduce.mul.v16i64(<16 x i64> %a0)767  ret i64 %1768}769 770;771; vXi32772;773 774define i32 @test_v2i32(<2 x i32> %a0) {775; SSE2-LABEL: test_v2i32:776; SSE2:       # %bb.0:777; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]778; SSE2-NEXT:    pmuludq %xmm0, %xmm1779; SSE2-NEXT:    movd %xmm1, %eax780; SSE2-NEXT:    retq781;782; SSE41-LABEL: test_v2i32:783; SSE41:       # %bb.0:784; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]785; SSE41-NEXT:    pmulld %xmm0, %xmm1786; SSE41-NEXT:    movd %xmm1, %eax787; SSE41-NEXT:    retq788;789; AVX-LABEL: test_v2i32:790; AVX:       # %bb.0:791; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]792; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0793; AVX-NEXT:    vmovd %xmm0, %eax794; AVX-NEXT:    retq795  %1 = call i32 @llvm.vector.reduce.mul.v2i32(<2 x i32> %a0)796  ret i32 %1797}798 799define i32 @test_v4i32(<4 x i32> %a0) {800; SSE2-LABEL: test_v4i32:801; SSE2:       # %bb.0:802; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]803; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]804; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]805; SSE2-NEXT:    pmuludq %xmm2, %xmm3806; SSE2-NEXT:    pmuludq %xmm0, %xmm1807; SSE2-NEXT:    pmuludq %xmm3, %xmm1808; SSE2-NEXT:    movd %xmm1, %eax809; SSE2-NEXT:    retq810;811; SSE41-LABEL: test_v4i32:812; SSE41:       # %bb.0:813; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]814; SSE41-NEXT:    pmulld %xmm0, %xmm1815; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]816; SSE41-NEXT:    pmulld %xmm1, %xmm0817; SSE41-NEXT:    movd %xmm0, %eax818; SSE41-NEXT:    retq819;820; AVX-LABEL: test_v4i32:821; AVX:       # %bb.0:822; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]823; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0824; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]825; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0826; AVX-NEXT:    vmovd %xmm0, %eax827; AVX-NEXT:    retq828  %1 = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> %a0)829  ret i32 %1830}831 832define i32 @test_v8i32(<8 x i32> %a0) {833; SSE2-LABEL: test_v8i32:834; SSE2:       # %bb.0:835; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]836; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]837; SSE2-NEXT:    pmuludq %xmm2, %xmm3838; SSE2-NEXT:    pmuludq %xmm1, %xmm0839; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]840; SSE2-NEXT:    pmuludq %xmm0, %xmm1841; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,2,2,2]842; SSE2-NEXT:    pmuludq %xmm3, %xmm0843; SSE2-NEXT:    pmuludq %xmm1, %xmm0844; SSE2-NEXT:    movd %xmm0, %eax845; SSE2-NEXT:    retq846;847; SSE41-LABEL: test_v8i32:848; SSE41:       # %bb.0:849; SSE41-NEXT:    pmulld %xmm1, %xmm0850; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]851; SSE41-NEXT:    pmulld %xmm0, %xmm1852; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]853; SSE41-NEXT:    pmulld %xmm1, %xmm0854; SSE41-NEXT:    movd %xmm0, %eax855; SSE41-NEXT:    retq856;857; AVX1-LABEL: test_v8i32:858; AVX1:       # %bb.0:859; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1860; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0861; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]862; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0863; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]864; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0865; AVX1-NEXT:    vmovd %xmm0, %eax866; AVX1-NEXT:    vzeroupper867; AVX1-NEXT:    retq868;869; AVX2-LABEL: test_v8i32:870; AVX2:       # %bb.0:871; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1872; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0873; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]874; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0875; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]876; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0877; AVX2-NEXT:    vmovd %xmm0, %eax878; AVX2-NEXT:    vzeroupper879; AVX2-NEXT:    retq880;881; AVX512-LABEL: test_v8i32:882; AVX512:       # %bb.0:883; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1884; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0885; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]886; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0887; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]888; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0889; AVX512-NEXT:    vmovd %xmm0, %eax890; AVX512-NEXT:    vzeroupper891; AVX512-NEXT:    retq892  %1 = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> %a0)893  ret i32 %1894}895 896define i32 @test_v16i32(<16 x i32> %a0) {897; SSE2-LABEL: test_v16i32:898; SSE2:       # %bb.0:899; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]900; SSE2-NEXT:    pmuludq %xmm3, %xmm1901; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]902; SSE2-NEXT:    pmuludq %xmm2, %xmm0903; SSE2-NEXT:    pmuludq %xmm1, %xmm0904; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]905; SSE2-NEXT:    pmuludq %xmm4, %xmm1906; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]907; SSE2-NEXT:    pmuludq %xmm5, %xmm2908; SSE2-NEXT:    pmuludq %xmm1, %xmm2909; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]910; SSE2-NEXT:    pmuludq %xmm0, %xmm1911; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]912; SSE2-NEXT:    pmuludq %xmm2, %xmm0913; SSE2-NEXT:    pmuludq %xmm1, %xmm0914; SSE2-NEXT:    movd %xmm0, %eax915; SSE2-NEXT:    retq916;917; SSE41-LABEL: test_v16i32:918; SSE41:       # %bb.0:919; SSE41-NEXT:    pmulld %xmm3, %xmm1920; SSE41-NEXT:    pmulld %xmm2, %xmm0921; SSE41-NEXT:    pmulld %xmm1, %xmm0922; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]923; SSE41-NEXT:    pmulld %xmm0, %xmm1924; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]925; SSE41-NEXT:    pmulld %xmm1, %xmm0926; SSE41-NEXT:    movd %xmm0, %eax927; SSE41-NEXT:    retq928;929; AVX1-LABEL: test_v16i32:930; AVX1:       # %bb.0:931; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2932; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3933; AVX1-NEXT:    vpmulld %xmm2, %xmm3, %xmm2934; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0935; AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm0936; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]937; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0938; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]939; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0940; AVX1-NEXT:    vmovd %xmm0, %eax941; AVX1-NEXT:    vzeroupper942; AVX1-NEXT:    retq943;944; AVX2-LABEL: test_v16i32:945; AVX2:       # %bb.0:946; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0947; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1948; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0949; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]950; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0951; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]952; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0953; AVX2-NEXT:    vmovd %xmm0, %eax954; AVX2-NEXT:    vzeroupper955; AVX2-NEXT:    retq956;957; AVX512-LABEL: test_v16i32:958; AVX512:       # %bb.0:959; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1960; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0961; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1962; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0963; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]964; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0965; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]966; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0967; AVX512-NEXT:    vmovd %xmm0, %eax968; AVX512-NEXT:    vzeroupper969; AVX512-NEXT:    retq970  %1 = call i32 @llvm.vector.reduce.mul.v16i32(<16 x i32> %a0)971  ret i32 %1972}973 974define i32 @test_v32i32(<32 x i32> %a0) {975; SSE2-LABEL: test_v32i32:976; SSE2:       # %bb.0:977; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[1,1,3,3]978; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm2[1,1,3,3]979; SSE2-NEXT:    pmuludq %xmm8, %xmm9980; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[1,1,3,3]981; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[1,1,3,3]982; SSE2-NEXT:    pmuludq %xmm8, %xmm10983; SSE2-NEXT:    pmuludq %xmm9, %xmm10984; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]985; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm3[1,1,3,3]986; SSE2-NEXT:    pmuludq %xmm8, %xmm9987; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[1,1,3,3]988; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,1,3,3]989; SSE2-NEXT:    pmuludq %xmm8, %xmm11990; SSE2-NEXT:    pmuludq %xmm9, %xmm11991; SSE2-NEXT:    pmuludq %xmm10, %xmm11992; SSE2-NEXT:    pmuludq %xmm6, %xmm2993; SSE2-NEXT:    pmuludq %xmm4, %xmm0994; SSE2-NEXT:    pmuludq %xmm2, %xmm0995; SSE2-NEXT:    pmuludq %xmm7, %xmm3996; SSE2-NEXT:    pmuludq %xmm5, %xmm1997; SSE2-NEXT:    pmuludq %xmm3, %xmm1998; SSE2-NEXT:    pmuludq %xmm0, %xmm1999; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]1000; SSE2-NEXT:    pmuludq %xmm1, %xmm01001; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[2,2,2,2]1002; SSE2-NEXT:    pmuludq %xmm11, %xmm11003; SSE2-NEXT:    pmuludq %xmm0, %xmm11004; SSE2-NEXT:    movd %xmm1, %eax1005; SSE2-NEXT:    retq1006;1007; SSE41-LABEL: test_v32i32:1008; SSE41:       # %bb.0:1009; SSE41-NEXT:    pmulld %xmm6, %xmm21010; SSE41-NEXT:    pmulld %xmm4, %xmm01011; SSE41-NEXT:    pmulld %xmm2, %xmm01012; SSE41-NEXT:    pmulld %xmm7, %xmm31013; SSE41-NEXT:    pmulld %xmm5, %xmm11014; SSE41-NEXT:    pmulld %xmm3, %xmm11015; SSE41-NEXT:    pmulld %xmm0, %xmm11016; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]1017; SSE41-NEXT:    pmulld %xmm1, %xmm01018; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1019; SSE41-NEXT:    pmulld %xmm0, %xmm11020; SSE41-NEXT:    movd %xmm1, %eax1021; SSE41-NEXT:    retq1022;1023; AVX1-LABEL: test_v32i32:1024; AVX1:       # %bb.0:1025; AVX1-NEXT:    vpmulld %xmm3, %xmm1, %xmm41026; AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm51027; AVX1-NEXT:    vpmulld %xmm4, %xmm5, %xmm41028; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm31029; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm11030; AVX1-NEXT:    vpmulld %xmm3, %xmm1, %xmm11031; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm21032; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm01033; AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm01034; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm01035; AVX1-NEXT:    vpmulld %xmm0, %xmm4, %xmm01036; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1037; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm01038; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1039; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm01040; AVX1-NEXT:    vmovd %xmm0, %eax1041; AVX1-NEXT:    vzeroupper1042; AVX1-NEXT:    retq1043;1044; AVX2-LABEL: test_v32i32:1045; AVX2:       # %bb.0:1046; AVX2-NEXT:    vpmulld %ymm3, %ymm1, %ymm11047; AVX2-NEXT:    vpmulld %ymm2, %ymm0, %ymm01048; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm01049; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm11050; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm01051; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1052; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm01053; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1054; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm01055; AVX2-NEXT:    vmovd %xmm0, %eax1056; AVX2-NEXT:    vzeroupper1057; AVX2-NEXT:    retq1058;1059; AVX512-LABEL: test_v32i32:1060; AVX512:       # %bb.0:1061; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm01062; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm11063; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm01064; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm11065; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm01066; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1067; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm01068; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1069; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm01070; AVX512-NEXT:    vmovd %xmm0, %eax1071; AVX512-NEXT:    vzeroupper1072; AVX512-NEXT:    retq1073  %1 = call i32 @llvm.vector.reduce.mul.v32i32(<32 x i32> %a0)1074  ret i32 %11075}1076 1077;1078; vXi161079;1080 1081define i16 @test_v2i16(<2 x i16> %a0) {1082; SSE-LABEL: test_v2i16:1083; SSE:       # %bb.0:1084; SSE-NEXT:    movdqa %xmm0, %xmm11085; SSE-NEXT:    psrld $16, %xmm11086; SSE-NEXT:    pmullw %xmm0, %xmm11087; SSE-NEXT:    movd %xmm1, %eax1088; SSE-NEXT:    # kill: def $ax killed $ax killed $eax1089; SSE-NEXT:    retq1090;1091; AVX-LABEL: test_v2i16:1092; AVX:       # %bb.0:1093; AVX-NEXT:    vpsrld $16, %xmm0, %xmm11094; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm01095; AVX-NEXT:    vmovd %xmm0, %eax1096; AVX-NEXT:    # kill: def $ax killed $ax killed $eax1097; AVX-NEXT:    retq1098  %1 = call i16 @llvm.vector.reduce.mul.v2i16(<2 x i16> %a0)1099  ret i16 %11100}1101 1102define i16 @test_v4i16(<4 x i16> %a0) {1103; SSE-LABEL: test_v4i16:1104; SSE:       # %bb.0:1105; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1106; SSE-NEXT:    pmullw %xmm0, %xmm11107; SSE-NEXT:    movdqa %xmm1, %xmm01108; SSE-NEXT:    psrld $16, %xmm01109; SSE-NEXT:    pmullw %xmm1, %xmm01110; SSE-NEXT:    movd %xmm0, %eax1111; SSE-NEXT:    # kill: def $ax killed $ax killed $eax1112; SSE-NEXT:    retq1113;1114; AVX-LABEL: test_v4i16:1115; AVX:       # %bb.0:1116; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1117; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm01118; AVX-NEXT:    vpsrld $16, %xmm0, %xmm11119; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm01120; AVX-NEXT:    vmovd %xmm0, %eax1121; AVX-NEXT:    # kill: def $ax killed $ax killed $eax1122; AVX-NEXT:    retq1123  %1 = call i16 @llvm.vector.reduce.mul.v4i16(<4 x i16> %a0)1124  ret i16 %11125}1126 1127define i16 @test_v8i16(<8 x i16> %a0) {1128; SSE-LABEL: test_v8i16:1129; SSE:       # %bb.0:1130; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1131; SSE-NEXT:    pmullw %xmm0, %xmm11132; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]1133; SSE-NEXT:    pmullw %xmm1, %xmm01134; SSE-NEXT:    movdqa %xmm0, %xmm11135; SSE-NEXT:    psrld $16, %xmm11136; SSE-NEXT:    pmullw %xmm0, %xmm11137; SSE-NEXT:    movd %xmm1, %eax1138; SSE-NEXT:    # kill: def $ax killed $ax killed $eax1139; SSE-NEXT:    retq1140;1141; AVX-LABEL: test_v8i16:1142; AVX:       # %bb.0:1143; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1144; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm01145; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1146; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm01147; AVX-NEXT:    vpsrld $16, %xmm0, %xmm11148; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm01149; AVX-NEXT:    vmovd %xmm0, %eax1150; AVX-NEXT:    # kill: def $ax killed $ax killed $eax1151; AVX-NEXT:    retq1152  %1 = call i16 @llvm.vector.reduce.mul.v8i16(<8 x i16> %a0)1153  ret i16 %11154}1155 1156define i16 @test_v16i16(<16 x i16> %a0) {1157; SSE-LABEL: test_v16i16:1158; SSE:       # %bb.0:1159; SSE-NEXT:    pmullw %xmm1, %xmm01160; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1161; SSE-NEXT:    pmullw %xmm0, %xmm11162; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]1163; SSE-NEXT:    pmullw %xmm1, %xmm01164; SSE-NEXT:    movdqa %xmm0, %xmm11165; SSE-NEXT:    psrld $16, %xmm11166; SSE-NEXT:    pmullw %xmm0, %xmm11167; SSE-NEXT:    movd %xmm1, %eax1168; SSE-NEXT:    # kill: def $ax killed $ax killed $eax1169; SSE-NEXT:    retq1170;1171; AVX1-LABEL: test_v16i16:1172; AVX1:       # %bb.0:1173; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm11174; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01175; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1176; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01177; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1178; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01179; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm11180; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01181; AVX1-NEXT:    vmovd %xmm0, %eax1182; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax1183; AVX1-NEXT:    vzeroupper1184; AVX1-NEXT:    retq1185;1186; AVX2-LABEL: test_v16i16:1187; AVX2:       # %bb.0:1188; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm11189; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01190; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1191; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01192; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1193; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01194; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm11195; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01196; AVX2-NEXT:    vmovd %xmm0, %eax1197; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax1198; AVX2-NEXT:    vzeroupper1199; AVX2-NEXT:    retq1200;1201; AVX512-LABEL: test_v16i16:1202; AVX512:       # %bb.0:1203; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm11204; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm01205; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1206; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm01207; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1208; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm01209; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm11210; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm01211; AVX512-NEXT:    vmovd %xmm0, %eax1212; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax1213; AVX512-NEXT:    vzeroupper1214; AVX512-NEXT:    retq1215  %1 = call i16 @llvm.vector.reduce.mul.v16i16(<16 x i16> %a0)1216  ret i16 %11217}1218 1219define i16 @test_v32i16(<32 x i16> %a0) {1220; SSE-LABEL: test_v32i16:1221; SSE:       # %bb.0:1222; SSE-NEXT:    pmullw %xmm3, %xmm11223; SSE-NEXT:    pmullw %xmm2, %xmm01224; SSE-NEXT:    pmullw %xmm1, %xmm01225; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1226; SSE-NEXT:    pmullw %xmm0, %xmm11227; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]1228; SSE-NEXT:    pmullw %xmm1, %xmm01229; SSE-NEXT:    movdqa %xmm0, %xmm11230; SSE-NEXT:    psrld $16, %xmm11231; SSE-NEXT:    pmullw %xmm0, %xmm11232; SSE-NEXT:    movd %xmm1, %eax1233; SSE-NEXT:    # kill: def $ax killed $ax killed $eax1234; SSE-NEXT:    retq1235;1236; AVX1-LABEL: test_v32i16:1237; AVX1:       # %bb.0:1238; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm21239; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm31240; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm21241; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01242; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm01243; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1244; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01245; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1246; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01247; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm11248; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01249; AVX1-NEXT:    vmovd %xmm0, %eax1250; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax1251; AVX1-NEXT:    vzeroupper1252; AVX1-NEXT:    retq1253;1254; AVX2-LABEL: test_v32i16:1255; AVX2:       # %bb.0:1256; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm01257; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm11258; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01259; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1260; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01261; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1262; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01263; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm11264; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01265; AVX2-NEXT:    vmovd %xmm0, %eax1266; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax1267; AVX2-NEXT:    vzeroupper1268; AVX2-NEXT:    retq1269;1270; AVX512BW-LABEL: test_v32i16:1271; AVX512BW:       # %bb.0:1272; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm11273; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm01274; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm11275; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm01276; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1277; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm01278; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1279; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm01280; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm11281; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm01282; AVX512BW-NEXT:    vmovd %xmm0, %eax1283; AVX512BW-NEXT:    # kill: def $ax killed $ax killed $eax1284; AVX512BW-NEXT:    vzeroupper1285; AVX512BW-NEXT:    retq1286;1287; AVX512BWVL-LABEL: test_v32i16:1288; AVX512BWVL:       # %bb.0:1289; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm11290; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm01291; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm11292; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01293; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1294; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01295; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1296; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01297; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm11298; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01299; AVX512BWVL-NEXT:    vmovd %xmm0, %eax1300; AVX512BWVL-NEXT:    # kill: def $ax killed $ax killed $eax1301; AVX512BWVL-NEXT:    vzeroupper1302; AVX512BWVL-NEXT:    retq1303;1304; AVX512DQ-LABEL: test_v32i16:1305; AVX512DQ:       # %bb.0:1306; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm11307; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm01308; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm11309; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm01310; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1311; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm01312; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1313; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm01314; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm11315; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm01316; AVX512DQ-NEXT:    vmovd %xmm0, %eax1317; AVX512DQ-NEXT:    # kill: def $ax killed $ax killed $eax1318; AVX512DQ-NEXT:    vzeroupper1319; AVX512DQ-NEXT:    retq1320;1321; AVX512DQVL-LABEL: test_v32i16:1322; AVX512DQVL:       # %bb.0:1323; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm11324; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm01325; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm11326; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01327; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1328; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01329; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1330; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01331; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm11332; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01333; AVX512DQVL-NEXT:    vmovd %xmm0, %eax1334; AVX512DQVL-NEXT:    # kill: def $ax killed $ax killed $eax1335; AVX512DQVL-NEXT:    vzeroupper1336; AVX512DQVL-NEXT:    retq1337  %1 = call i16 @llvm.vector.reduce.mul.v32i16(<32 x i16> %a0)1338  ret i16 %11339}1340 1341define i16 @test_v64i16(<64 x i16> %a0) {1342; SSE-LABEL: test_v64i16:1343; SSE:       # %bb.0:1344; SSE-NEXT:    pmullw %xmm6, %xmm21345; SSE-NEXT:    pmullw %xmm4, %xmm01346; SSE-NEXT:    pmullw %xmm2, %xmm01347; SSE-NEXT:    pmullw %xmm7, %xmm31348; SSE-NEXT:    pmullw %xmm5, %xmm11349; SSE-NEXT:    pmullw %xmm3, %xmm11350; SSE-NEXT:    pmullw %xmm0, %xmm11351; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]1352; SSE-NEXT:    pmullw %xmm1, %xmm01353; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1354; SSE-NEXT:    pmullw %xmm0, %xmm11355; SSE-NEXT:    movdqa %xmm1, %xmm01356; SSE-NEXT:    psrld $16, %xmm01357; SSE-NEXT:    pmullw %xmm1, %xmm01358; SSE-NEXT:    movd %xmm0, %eax1359; SSE-NEXT:    # kill: def $ax killed $ax killed $eax1360; SSE-NEXT:    retq1361;1362; AVX1-LABEL: test_v64i16:1363; AVX1:       # %bb.0:1364; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm41365; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm51366; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm41367; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm31368; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm11369; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm11370; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm21371; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm01372; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm01373; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01374; AVX1-NEXT:    vpmullw %xmm0, %xmm4, %xmm01375; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1376; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01377; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1378; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01379; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm11380; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01381; AVX1-NEXT:    vmovd %xmm0, %eax1382; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax1383; AVX1-NEXT:    vzeroupper1384; AVX1-NEXT:    retq1385;1386; AVX2-LABEL: test_v64i16:1387; AVX2:       # %bb.0:1388; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm11389; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm01390; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm01391; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm11392; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01393; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1394; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01395; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1396; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01397; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm11398; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01399; AVX2-NEXT:    vmovd %xmm0, %eax1400; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax1401; AVX2-NEXT:    vzeroupper1402; AVX2-NEXT:    retq1403;1404; AVX512BW-LABEL: test_v64i16:1405; AVX512BW:       # %bb.0:1406; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm01407; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm11408; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm01409; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm11410; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm01411; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1412; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm01413; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1414; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm01415; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm11416; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm01417; AVX512BW-NEXT:    vmovd %xmm0, %eax1418; AVX512BW-NEXT:    # kill: def $ax killed $ax killed $eax1419; AVX512BW-NEXT:    vzeroupper1420; AVX512BW-NEXT:    retq1421;1422; AVX512BWVL-LABEL: test_v64i16:1423; AVX512BWVL:       # %bb.0:1424; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm01425; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm11426; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm01427; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm11428; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01429; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1430; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01431; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1432; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01433; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm11434; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01435; AVX512BWVL-NEXT:    vmovd %xmm0, %eax1436; AVX512BWVL-NEXT:    # kill: def $ax killed $ax killed $eax1437; AVX512BWVL-NEXT:    vzeroupper1438; AVX512BWVL-NEXT:    retq1439;1440; AVX512DQ-LABEL: test_v64i16:1441; AVX512DQ:       # %bb.0:1442; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm21443; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm31444; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm3, %ymm21445; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm01446; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm0, %ymm01447; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm11448; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm01449; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1450; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm01451; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1452; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm01453; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm11454; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm01455; AVX512DQ-NEXT:    vmovd %xmm0, %eax1456; AVX512DQ-NEXT:    # kill: def $ax killed $ax killed $eax1457; AVX512DQ-NEXT:    vzeroupper1458; AVX512DQ-NEXT:    retq1459;1460; AVX512DQVL-LABEL: test_v64i16:1461; AVX512DQVL:       # %bb.0:1462; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm1, %ymm21463; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm31464; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm3, %ymm21465; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm01466; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm01467; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm11468; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01469; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1470; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01471; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1472; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01473; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm11474; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01475; AVX512DQVL-NEXT:    vmovd %xmm0, %eax1476; AVX512DQVL-NEXT:    # kill: def $ax killed $ax killed $eax1477; AVX512DQVL-NEXT:    vzeroupper1478; AVX512DQVL-NEXT:    retq1479  %1 = call i16 @llvm.vector.reduce.mul.v64i16(<64 x i16> %a0)1480  ret i16 %11481}1482 1483;1484; vXi81485;1486 1487define i8 @test_v2i8(<2 x i8> %a0) {1488; SSE-LABEL: test_v2i8:1489; SSE:       # %bb.0:1490; SSE-NEXT:    movdqa %xmm0, %xmm11491; SSE-NEXT:    psrlw $8, %xmm11492; SSE-NEXT:    pmullw %xmm0, %xmm11493; SSE-NEXT:    movd %xmm1, %eax1494; SSE-NEXT:    # kill: def $al killed $al killed $eax1495; SSE-NEXT:    retq1496;1497; AVX-LABEL: test_v2i8:1498; AVX:       # %bb.0:1499; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm11500; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm01501; AVX-NEXT:    vmovd %xmm0, %eax1502; AVX-NEXT:    # kill: def $al killed $al killed $eax1503; AVX-NEXT:    retq1504  %1 = call i8 @llvm.vector.reduce.mul.v2i8(<2 x i8> %a0)1505  ret i8 %11506}1507 1508define i8 @test_v4i8(<4 x i8> %a0) {1509; SSE2-LABEL: test_v4i8:1510; SSE2:       # %bb.0:1511; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1512; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1513; SSE2-NEXT:    pmullw %xmm0, %xmm11514; SSE2-NEXT:    movdqa %xmm1, %xmm01515; SSE2-NEXT:    psrld $16, %xmm01516; SSE2-NEXT:    pmullw %xmm1, %xmm01517; SSE2-NEXT:    movd %xmm0, %eax1518; SSE2-NEXT:    # kill: def $al killed $al killed $eax1519; SSE2-NEXT:    retq1520;1521; SSE41-LABEL: test_v4i8:1522; SSE41:       # %bb.0:1523; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1524; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1525; SSE41-NEXT:    pmullw %xmm0, %xmm11526; SSE41-NEXT:    movdqa %xmm1, %xmm01527; SSE41-NEXT:    psrld $16, %xmm01528; SSE41-NEXT:    pmullw %xmm1, %xmm01529; SSE41-NEXT:    movd %xmm0, %eax1530; SSE41-NEXT:    # kill: def $al killed $al killed $eax1531; SSE41-NEXT:    retq1532;1533; AVX-LABEL: test_v4i8:1534; AVX:       # %bb.0:1535; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1536; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1537; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm01538; AVX-NEXT:    vpsrld $16, %xmm0, %xmm11539; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm01540; AVX-NEXT:    vmovd %xmm0, %eax1541; AVX-NEXT:    # kill: def $al killed $al killed $eax1542; AVX-NEXT:    retq1543  %1 = call i8 @llvm.vector.reduce.mul.v4i8(<4 x i8> %a0)1544  ret i8 %11545}1546 1547define i8 @test_v8i8(<8 x i8> %a0) {1548; SSE2-LABEL: test_v8i8:1549; SSE2:       # %bb.0:1550; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1551; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1552; SSE2-NEXT:    pmullw %xmm0, %xmm11553; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]1554; SSE2-NEXT:    pmullw %xmm1, %xmm01555; SSE2-NEXT:    movdqa %xmm0, %xmm11556; SSE2-NEXT:    psrld $16, %xmm11557; SSE2-NEXT:    pmullw %xmm0, %xmm11558; SSE2-NEXT:    movd %xmm1, %eax1559; SSE2-NEXT:    # kill: def $al killed $al killed $eax1560; SSE2-NEXT:    retq1561;1562; SSE41-LABEL: test_v8i8:1563; SSE41:       # %bb.0:1564; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1565; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1566; SSE41-NEXT:    pmullw %xmm0, %xmm11567; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]1568; SSE41-NEXT:    pmullw %xmm1, %xmm01569; SSE41-NEXT:    movdqa %xmm0, %xmm11570; SSE41-NEXT:    psrld $16, %xmm11571; SSE41-NEXT:    pmullw %xmm0, %xmm11572; SSE41-NEXT:    movd %xmm1, %eax1573; SSE41-NEXT:    # kill: def $al killed $al killed $eax1574; SSE41-NEXT:    retq1575;1576; AVX-LABEL: test_v8i8:1577; AVX:       # %bb.0:1578; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1579; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1580; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm01581; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1582; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm01583; AVX-NEXT:    vpsrld $16, %xmm0, %xmm11584; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm01585; AVX-NEXT:    vmovd %xmm0, %eax1586; AVX-NEXT:    # kill: def $al killed $al killed $eax1587; AVX-NEXT:    retq1588  %1 = call i8 @llvm.vector.reduce.mul.v8i8(<8 x i8> %a0)1589  ret i8 %11590}1591 1592define i8 @test_v16i8(<16 x i8> %a0) {1593; SSE2-LABEL: test_v16i8:1594; SSE2:       # %bb.0:1595; SSE2-NEXT:    movdqa %xmm0, %xmm11596; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1597; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1598; SSE2-NEXT:    pmullw %xmm1, %xmm01599; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1600; SSE2-NEXT:    pmullw %xmm0, %xmm11601; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]1602; SSE2-NEXT:    pmullw %xmm1, %xmm01603; SSE2-NEXT:    movdqa %xmm0, %xmm11604; SSE2-NEXT:    psrld $16, %xmm11605; SSE2-NEXT:    pmullw %xmm0, %xmm11606; SSE2-NEXT:    movd %xmm1, %eax1607; SSE2-NEXT:    # kill: def $al killed $al killed $eax1608; SSE2-NEXT:    retq1609;1610; SSE41-LABEL: test_v16i8:1611; SSE41:       # %bb.0:1612; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1613; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1614; SSE41-NEXT:    pmullw %xmm1, %xmm01615; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1616; SSE41-NEXT:    pmullw %xmm0, %xmm11617; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]1618; SSE41-NEXT:    pmullw %xmm1, %xmm01619; SSE41-NEXT:    movdqa %xmm0, %xmm11620; SSE41-NEXT:    psrld $16, %xmm11621; SSE41-NEXT:    pmullw %xmm0, %xmm11622; SSE41-NEXT:    movd %xmm1, %eax1623; SSE41-NEXT:    # kill: def $al killed $al killed $eax1624; SSE41-NEXT:    retq1625;1626; AVX-LABEL: test_v16i8:1627; AVX:       # %bb.0:1628; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1629; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1630; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm01631; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1632; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm01633; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1634; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm01635; AVX-NEXT:    vpsrld $16, %xmm0, %xmm11636; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm01637; AVX-NEXT:    vmovd %xmm0, %eax1638; AVX-NEXT:    # kill: def $al killed $al killed $eax1639; AVX-NEXT:    retq1640  %1 = call i8 @llvm.vector.reduce.mul.v16i8(<16 x i8> %a0)1641  ret i8 %11642}1643 1644define i8 @test_v32i8(<32 x i8> %a0) {1645; SSE2-LABEL: test_v32i8:1646; SSE2:       # %bb.0:1647; SSE2-NEXT:    movdqa %xmm1, %xmm21648; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1649; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1650; SSE2-NEXT:    pmullw %xmm2, %xmm11651; SSE2-NEXT:    movdqa %xmm0, %xmm21652; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1653; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1654; SSE2-NEXT:    pmullw %xmm2, %xmm01655; SSE2-NEXT:    pmullw %xmm1, %xmm01656; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1657; SSE2-NEXT:    pmullw %xmm0, %xmm11658; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]1659; SSE2-NEXT:    pmullw %xmm1, %xmm01660; SSE2-NEXT:    movdqa %xmm0, %xmm11661; SSE2-NEXT:    psrld $16, %xmm11662; SSE2-NEXT:    pmullw %xmm0, %xmm11663; SSE2-NEXT:    movd %xmm1, %eax1664; SSE2-NEXT:    # kill: def $al killed $al killed $eax1665; SSE2-NEXT:    retq1666;1667; SSE41-LABEL: test_v32i8:1668; SSE41:       # %bb.0:1669; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1670; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1671; SSE41-NEXT:    pmullw %xmm2, %xmm11672; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1673; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1674; SSE41-NEXT:    pmullw %xmm2, %xmm01675; SSE41-NEXT:    pmullw %xmm1, %xmm01676; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1677; SSE41-NEXT:    pmullw %xmm0, %xmm11678; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]1679; SSE41-NEXT:    pmullw %xmm1, %xmm01680; SSE41-NEXT:    movdqa %xmm0, %xmm11681; SSE41-NEXT:    psrld $16, %xmm11682; SSE41-NEXT:    pmullw %xmm0, %xmm11683; SSE41-NEXT:    movd %xmm1, %eax1684; SSE41-NEXT:    # kill: def $al killed $al killed $eax1685; SSE41-NEXT:    retq1686;1687; AVX1-LABEL: test_v32i8:1688; AVX1:       # %bb.0:1689; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm11690; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1691; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1692; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm11693; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1694; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1695; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm01696; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01697; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1698; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01699; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1700; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01701; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm11702; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01703; AVX1-NEXT:    vmovd %xmm0, %eax1704; AVX1-NEXT:    # kill: def $al killed $al killed $eax1705; AVX1-NEXT:    vzeroupper1706; AVX1-NEXT:    retq1707;1708; AVX2-LABEL: test_v32i8:1709; AVX2:       # %bb.0:1710; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]1711; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]1712; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm01713; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm11714; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01715; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1716; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01717; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1718; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01719; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm11720; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01721; AVX2-NEXT:    vmovd %xmm0, %eax1722; AVX2-NEXT:    # kill: def $al killed $al killed $eax1723; AVX2-NEXT:    vzeroupper1724; AVX2-NEXT:    retq1725;1726; AVX512-LABEL: test_v32i8:1727; AVX512:       # %bb.0:1728; AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]1729; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]1730; AVX512-NEXT:    vpmullw %ymm1, %ymm0, %ymm01731; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm11732; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm01733; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1734; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm01735; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1736; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm01737; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm11738; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm01739; AVX512-NEXT:    vmovd %xmm0, %eax1740; AVX512-NEXT:    # kill: def $al killed $al killed $eax1741; AVX512-NEXT:    vzeroupper1742; AVX512-NEXT:    retq1743  %1 = call i8 @llvm.vector.reduce.mul.v32i8(<32 x i8> %a0)1744  ret i8 %11745}1746 1747define i8 @test_v64i8(<64 x i8> %a0) {1748; SSE2-LABEL: test_v64i8:1749; SSE2:       # %bb.0:1750; SSE2-NEXT:    movdqa %xmm3, %xmm41751; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1752; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1753; SSE2-NEXT:    pmullw %xmm4, %xmm31754; SSE2-NEXT:    movdqa %xmm1, %xmm41755; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1756; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1757; SSE2-NEXT:    pmullw %xmm4, %xmm11758; SSE2-NEXT:    pmullw %xmm3, %xmm11759; SSE2-NEXT:    movdqa %xmm2, %xmm31760; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1761; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1762; SSE2-NEXT:    pmullw %xmm3, %xmm21763; SSE2-NEXT:    movdqa %xmm0, %xmm31764; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1765; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1766; SSE2-NEXT:    pmullw %xmm3, %xmm01767; SSE2-NEXT:    pmullw %xmm2, %xmm01768; SSE2-NEXT:    pmullw %xmm1, %xmm01769; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1770; SSE2-NEXT:    pmullw %xmm0, %xmm11771; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]1772; SSE2-NEXT:    pmullw %xmm1, %xmm01773; SSE2-NEXT:    movdqa %xmm0, %xmm11774; SSE2-NEXT:    psrld $16, %xmm11775; SSE2-NEXT:    pmullw %xmm0, %xmm11776; SSE2-NEXT:    movd %xmm1, %eax1777; SSE2-NEXT:    # kill: def $al killed $al killed $eax1778; SSE2-NEXT:    retq1779;1780; SSE41-LABEL: test_v64i8:1781; SSE41:       # %bb.0:1782; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero1783; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1784; SSE41-NEXT:    pmullw %xmm4, %xmm31785; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1786; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1787; SSE41-NEXT:    pmullw %xmm4, %xmm11788; SSE41-NEXT:    pmullw %xmm3, %xmm11789; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero1790; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1791; SSE41-NEXT:    pmullw %xmm3, %xmm21792; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1793; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1794; SSE41-NEXT:    pmullw %xmm3, %xmm01795; SSE41-NEXT:    pmullw %xmm2, %xmm01796; SSE41-NEXT:    pmullw %xmm1, %xmm01797; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1798; SSE41-NEXT:    pmullw %xmm0, %xmm11799; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]1800; SSE41-NEXT:    pmullw %xmm1, %xmm01801; SSE41-NEXT:    movdqa %xmm0, %xmm11802; SSE41-NEXT:    psrld $16, %xmm11803; SSE41-NEXT:    pmullw %xmm0, %xmm11804; SSE41-NEXT:    movd %xmm1, %eax1805; SSE41-NEXT:    # kill: def $al killed $al killed $eax1806; SSE41-NEXT:    retq1807;1808; AVX1-LABEL: test_v64i8:1809; AVX1:       # %bb.0:1810; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm21811; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1812; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero1813; AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm21814; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm31815; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1816; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero1817; AVX1-NEXT:    vpmullw %xmm4, %xmm3, %xmm31818; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm21819; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1820; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1821; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm11822; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1823; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1824; AVX1-NEXT:    vpmullw %xmm3, %xmm0, %xmm01825; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01826; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm01827; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1828; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01829; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1830; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01831; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm11832; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01833; AVX1-NEXT:    vmovd %xmm0, %eax1834; AVX1-NEXT:    # kill: def $al killed $al killed $eax1835; AVX1-NEXT:    vzeroupper1836; AVX1-NEXT:    retq1837;1838; AVX2-LABEL: test_v64i8:1839; AVX2:       # %bb.0:1840; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]1841; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]1842; AVX2-NEXT:    vpmullw %ymm2, %ymm1, %ymm11843; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]1844; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]1845; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm01846; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm01847; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm11848; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01849; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1850; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01851; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1852; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01853; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm11854; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm01855; AVX2-NEXT:    vmovd %xmm0, %eax1856; AVX2-NEXT:    # kill: def $al killed $al killed $eax1857; AVX2-NEXT:    vzeroupper1858; AVX2-NEXT:    retq1859;1860; AVX512BW-LABEL: test_v64i8:1861; AVX512BW:       # %bb.0:1862; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]1863; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]1864; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm01865; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm11866; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm01867; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm11868; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm01869; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1870; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm01871; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1872; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm01873; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm11874; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm01875; AVX512BW-NEXT:    vmovd %xmm0, %eax1876; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax1877; AVX512BW-NEXT:    vzeroupper1878; AVX512BW-NEXT:    retq1879;1880; AVX512BWVL-LABEL: test_v64i8:1881; AVX512BWVL:       # %bb.0:1882; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]1883; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]1884; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm01885; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm11886; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm01887; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm11888; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01889; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1890; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01891; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1892; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01893; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm11894; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01895; AVX512BWVL-NEXT:    vmovd %xmm0, %eax1896; AVX512BWVL-NEXT:    # kill: def $al killed $al killed $eax1897; AVX512BWVL-NEXT:    vzeroupper1898; AVX512BWVL-NEXT:    retq1899;1900; AVX512DQ-LABEL: test_v64i8:1901; AVX512DQ:       # %bb.0:1902; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm11903; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]1904; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]1905; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm1, %ymm11906; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]1907; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]1908; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm0, %ymm01909; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm01910; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm11911; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm01912; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1913; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm01914; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1915; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm01916; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm11917; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm01918; AVX512DQ-NEXT:    vmovd %xmm0, %eax1919; AVX512DQ-NEXT:    # kill: def $al killed $al killed $eax1920; AVX512DQ-NEXT:    vzeroupper1921; AVX512DQ-NEXT:    retq1922;1923; AVX512DQVL-LABEL: test_v64i8:1924; AVX512DQVL:       # %bb.0:1925; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm11926; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]1927; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]1928; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm1, %ymm11929; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]1930; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]1931; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm01932; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm01933; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm11934; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01935; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1936; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01937; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]1938; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01939; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm11940; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm01941; AVX512DQVL-NEXT:    vmovd %xmm0, %eax1942; AVX512DQVL-NEXT:    # kill: def $al killed $al killed $eax1943; AVX512DQVL-NEXT:    vzeroupper1944; AVX512DQVL-NEXT:    retq1945  %1 = call i8 @llvm.vector.reduce.mul.v64i8(<64 x i8> %a0)1946  ret i8 %11947}1948 1949define i8 @test_v128i8(<128 x i8> %a0) {1950; SSE2-LABEL: test_v128i8:1951; SSE2:       # %bb.0:1952; SSE2-NEXT:    movdqa %xmm7, %xmm81953; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1954; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1955; SSE2-NEXT:    pmullw %xmm8, %xmm71956; SSE2-NEXT:    movdqa %xmm3, %xmm81957; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1958; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1959; SSE2-NEXT:    pmullw %xmm8, %xmm31960; SSE2-NEXT:    pmullw %xmm7, %xmm31961; SSE2-NEXT:    movdqa %xmm5, %xmm71962; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1963; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1964; SSE2-NEXT:    pmullw %xmm7, %xmm51965; SSE2-NEXT:    movdqa %xmm1, %xmm71966; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1967; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1968; SSE2-NEXT:    pmullw %xmm7, %xmm11969; SSE2-NEXT:    pmullw %xmm5, %xmm11970; SSE2-NEXT:    pmullw %xmm3, %xmm11971; SSE2-NEXT:    movdqa %xmm6, %xmm31972; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1973; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1974; SSE2-NEXT:    pmullw %xmm3, %xmm61975; SSE2-NEXT:    movdqa %xmm2, %xmm31976; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1977; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1978; SSE2-NEXT:    pmullw %xmm3, %xmm21979; SSE2-NEXT:    pmullw %xmm6, %xmm21980; SSE2-NEXT:    movdqa %xmm4, %xmm31981; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1982; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1983; SSE2-NEXT:    pmullw %xmm3, %xmm41984; SSE2-NEXT:    movdqa %xmm0, %xmm31985; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1986; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1987; SSE2-NEXT:    pmullw %xmm3, %xmm01988; SSE2-NEXT:    pmullw %xmm4, %xmm01989; SSE2-NEXT:    pmullw %xmm2, %xmm01990; SSE2-NEXT:    pmullw %xmm1, %xmm01991; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]1992; SSE2-NEXT:    pmullw %xmm0, %xmm11993; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]1994; SSE2-NEXT:    pmullw %xmm1, %xmm01995; SSE2-NEXT:    movdqa %xmm0, %xmm11996; SSE2-NEXT:    psrld $16, %xmm11997; SSE2-NEXT:    pmullw %xmm0, %xmm11998; SSE2-NEXT:    movd %xmm1, %eax1999; SSE2-NEXT:    # kill: def $al killed $al killed $eax2000; SSE2-NEXT:    retq2001;2002; SSE41-LABEL: test_v128i8:2003; SSE41:       # %bb.0:2004; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm8 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero2005; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2006; SSE41-NEXT:    pmullw %xmm8, %xmm72007; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm8 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero2008; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2009; SSE41-NEXT:    pmullw %xmm8, %xmm32010; SSE41-NEXT:    pmullw %xmm7, %xmm32011; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero2012; SSE41-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2013; SSE41-NEXT:    pmullw %xmm7, %xmm52014; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm7 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero2015; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2016; SSE41-NEXT:    pmullw %xmm7, %xmm12017; SSE41-NEXT:    pmullw %xmm5, %xmm12018; SSE41-NEXT:    pmullw %xmm3, %xmm12019; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero2020; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2021; SSE41-NEXT:    pmullw %xmm3, %xmm62022; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero2023; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2024; SSE41-NEXT:    pmullw %xmm3, %xmm22025; SSE41-NEXT:    pmullw %xmm6, %xmm22026; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero2027; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2028; SSE41-NEXT:    pmullw %xmm3, %xmm42029; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero2030; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2031; SSE41-NEXT:    pmullw %xmm3, %xmm02032; SSE41-NEXT:    pmullw %xmm4, %xmm02033; SSE41-NEXT:    pmullw %xmm2, %xmm02034; SSE41-NEXT:    pmullw %xmm1, %xmm02035; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]2036; SSE41-NEXT:    pmullw %xmm0, %xmm12037; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]2038; SSE41-NEXT:    pmullw %xmm1, %xmm02039; SSE41-NEXT:    movdqa %xmm0, %xmm12040; SSE41-NEXT:    psrld $16, %xmm12041; SSE41-NEXT:    pmullw %xmm0, %xmm12042; SSE41-NEXT:    movd %xmm1, %eax2043; SSE41-NEXT:    # kill: def $al killed $al killed $eax2044; SSE41-NEXT:    retq2045;2046; AVX1-LABEL: test_v128i8:2047; AVX1:       # %bb.0:2048; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm42049; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2050; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero2051; AVX1-NEXT:    vpmullw %xmm5, %xmm4, %xmm42052; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm52053; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2054; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero2055; AVX1-NEXT:    vpmullw %xmm6, %xmm5, %xmm52056; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm42057; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm52058; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2059; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero2060; AVX1-NEXT:    vpmullw %xmm6, %xmm5, %xmm52061; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm62062; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2063; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero2064; AVX1-NEXT:    vpmullw %xmm7, %xmm6, %xmm62065; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm52066; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm42067; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2068; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero2069; AVX1-NEXT:    vpmullw %xmm5, %xmm3, %xmm32070; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2071; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero2072; AVX1-NEXT:    vpmullw %xmm5, %xmm1, %xmm12073; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm12074; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2075; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero2076; AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm22077; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2078; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero2079; AVX1-NEXT:    vpmullw %xmm3, %xmm0, %xmm02080; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm02081; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm02082; AVX1-NEXT:    vpmullw %xmm4, %xmm0, %xmm02083; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]2084; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm02085; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]2086; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm02087; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm12088; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm02089; AVX1-NEXT:    vmovd %xmm0, %eax2090; AVX1-NEXT:    # kill: def $al killed $al killed $eax2091; AVX1-NEXT:    vzeroupper2092; AVX1-NEXT:    retq2093;2094; AVX2-LABEL: test_v128i8:2095; AVX2:       # %bb.0:2096; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]2097; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]2098; AVX2-NEXT:    vpmullw %ymm4, %ymm3, %ymm32099; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]2100; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]2101; AVX2-NEXT:    vpmullw %ymm4, %ymm1, %ymm12102; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm12103; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]2104; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]2105; AVX2-NEXT:    vpmullw %ymm3, %ymm2, %ymm22106; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]2107; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]2108; AVX2-NEXT:    vpmullw %ymm3, %ymm0, %ymm02109; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm02110; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm02111; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm12112; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm02113; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]2114; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm02115; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]2116; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm02117; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm12118; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm02119; AVX2-NEXT:    vmovd %xmm0, %eax2120; AVX2-NEXT:    # kill: def $al killed $al killed $eax2121; AVX2-NEXT:    vzeroupper2122; AVX2-NEXT:    retq2123;2124; AVX512BW-LABEL: test_v128i8:2125; AVX512BW:       # %bb.0:2126; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]2127; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]2128; AVX512BW-NEXT:    vpmullw %zmm2, %zmm1, %zmm12129; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]2130; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]2131; AVX512BW-NEXT:    vpmullw %zmm2, %zmm0, %zmm02132; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm02133; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm12134; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm02135; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm12136; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm02137; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]2138; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm02139; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]2140; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm02141; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm12142; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm02143; AVX512BW-NEXT:    vmovd %xmm0, %eax2144; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax2145; AVX512BW-NEXT:    vzeroupper2146; AVX512BW-NEXT:    retq2147;2148; AVX512BWVL-LABEL: test_v128i8:2149; AVX512BWVL:       # %bb.0:2150; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]2151; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]2152; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm1, %zmm12153; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]2154; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]2155; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm0, %zmm02156; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm02157; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm12158; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm02159; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm12160; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm02161; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]2162; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm02163; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]2164; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm02165; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm12166; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm02167; AVX512BWVL-NEXT:    vmovd %xmm0, %eax2168; AVX512BWVL-NEXT:    # kill: def $al killed $al killed $eax2169; AVX512BWVL-NEXT:    vzeroupper2170; AVX512BWVL-NEXT:    retq2171;2172; AVX512DQ-LABEL: test_v128i8:2173; AVX512DQ:       # %bb.0:2174; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm22175; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]2176; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]2177; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm2, %ymm22178; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm32179; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]2180; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]2181; AVX512DQ-NEXT:    vpmullw %ymm4, %ymm3, %ymm32182; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm3, %ymm22183; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]2184; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]2185; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm1, %ymm12186; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]2187; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]2188; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm0, %ymm02189; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm02190; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm0, %ymm02191; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm12192; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm02193; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]2194; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm02195; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]2196; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm02197; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm12198; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm02199; AVX512DQ-NEXT:    vmovd %xmm0, %eax2200; AVX512DQ-NEXT:    # kill: def $al killed $al killed $eax2201; AVX512DQ-NEXT:    vzeroupper2202; AVX512DQ-NEXT:    retq2203;2204; AVX512DQVL-LABEL: test_v128i8:2205; AVX512DQVL:       # %bb.0:2206; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm1, %ymm22207; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]2208; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]2209; AVX512DQVL-NEXT:    vpmullw %ymm3, %ymm2, %ymm22210; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm32211; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]2212; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]2213; AVX512DQVL-NEXT:    vpmullw %ymm4, %ymm3, %ymm32214; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm3, %ymm22215; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]2216; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]2217; AVX512DQVL-NEXT:    vpmullw %ymm3, %ymm1, %ymm12218; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]2219; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]2220; AVX512DQVL-NEXT:    vpmullw %ymm3, %ymm0, %ymm02221; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm02222; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm02223; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm12224; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm02225; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]2226; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm02227; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]2228; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm02229; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm12230; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm02231; AVX512DQVL-NEXT:    vmovd %xmm0, %eax2232; AVX512DQVL-NEXT:    # kill: def $al killed $al killed $eax2233; AVX512DQVL-NEXT:    vzeroupper2234; AVX512DQVL-NEXT:    retq2235  %1 = call i8 @llvm.vector.reduce.mul.v128i8(<128 x i8> %a0)2236  ret i8 %12237}2238 2239;2240; Legalization2241;2242 2243define i8 @illegal_v4i8(i8 %a0, ptr %a1) {2244; SSE2-LABEL: illegal_v4i8:2245; SSE2:       # %bb.0:2246; SSE2-NEXT:    movl %edi, %eax2247; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero2248; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]2249; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]2250; SSE2-NEXT:    pmullw %xmm0, %xmm12251; SSE2-NEXT:    movdqa %xmm1, %xmm02252; SSE2-NEXT:    psrld $16, %xmm02253; SSE2-NEXT:    pmullw %xmm1, %xmm02254; SSE2-NEXT:    movd %xmm0, %ecx2255; SSE2-NEXT:    # kill: def $al killed $al killed $eax2256; SSE2-NEXT:    mulb %cl2257; SSE2-NEXT:    retq2258;2259; SSE41-LABEL: illegal_v4i8:2260; SSE41:       # %bb.0:2261; SSE41-NEXT:    movl %edi, %eax2262; SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero2263; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero2264; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]2265; SSE41-NEXT:    pmullw %xmm0, %xmm12266; SSE41-NEXT:    movdqa %xmm1, %xmm02267; SSE41-NEXT:    psrld $16, %xmm02268; SSE41-NEXT:    pmullw %xmm1, %xmm02269; SSE41-NEXT:    movd %xmm0, %ecx2270; SSE41-NEXT:    # kill: def $al killed $al killed $eax2271; SSE41-NEXT:    mulb %cl2272; SSE41-NEXT:    retq2273;2274; AVX-LABEL: illegal_v4i8:2275; AVX:       # %bb.0:2276; AVX-NEXT:    movl %edi, %eax2277; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero2278; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero2279; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]2280; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm02281; AVX-NEXT:    vpsrld $16, %xmm0, %xmm12282; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm02283; AVX-NEXT:    vmovd %xmm0, %ecx2284; AVX-NEXT:    # kill: def $al killed $al killed $eax2285; AVX-NEXT:    mulb %cl2286; AVX-NEXT:    retq2287  %ld = load <4 x i8>, ptr %a1, align 42288  %rdx = call i8 @llvm.vector.reduce.mul.v4i8(<4 x i8> %ld)2289  %mul = mul i8 %a0, %rdx2290  ret i8 %mul2291}2292 2293define i8 @illegal_v8i8(i8 %a0, ptr %a1) {2294; SSE2-LABEL: illegal_v8i8:2295; SSE2:       # %bb.0:2296; SSE2-NEXT:    movl %edi, %eax2297; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero2298; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]2299; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]2300; SSE2-NEXT:    pmullw %xmm0, %xmm12301; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]2302; SSE2-NEXT:    pmullw %xmm1, %xmm02303; SSE2-NEXT:    movdqa %xmm0, %xmm12304; SSE2-NEXT:    psrld $16, %xmm12305; SSE2-NEXT:    pmullw %xmm0, %xmm12306; SSE2-NEXT:    movd %xmm1, %ecx2307; SSE2-NEXT:    # kill: def $al killed $al killed $eax2308; SSE2-NEXT:    mulb %cl2309; SSE2-NEXT:    retq2310;2311; SSE41-LABEL: illegal_v8i8:2312; SSE41:       # %bb.0:2313; SSE41-NEXT:    movl %edi, %eax2314; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero2315; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]2316; SSE41-NEXT:    pmullw %xmm0, %xmm12317; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]2318; SSE41-NEXT:    pmullw %xmm1, %xmm02319; SSE41-NEXT:    movdqa %xmm0, %xmm12320; SSE41-NEXT:    psrld $16, %xmm12321; SSE41-NEXT:    pmullw %xmm0, %xmm12322; SSE41-NEXT:    movd %xmm1, %ecx2323; SSE41-NEXT:    # kill: def $al killed $al killed $eax2324; SSE41-NEXT:    mulb %cl2325; SSE41-NEXT:    retq2326;2327; AVX-LABEL: illegal_v8i8:2328; AVX:       # %bb.0:2329; AVX-NEXT:    movl %edi, %eax2330; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero2331; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]2332; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm02333; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]2334; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm02335; AVX-NEXT:    vpsrld $16, %xmm0, %xmm12336; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm02337; AVX-NEXT:    vmovd %xmm0, %ecx2338; AVX-NEXT:    # kill: def $al killed $al killed $eax2339; AVX-NEXT:    mulb %cl2340; AVX-NEXT:    retq2341  %ld = load <8 x i8>, ptr %a1, align 42342  %rdx = call i8 @llvm.vector.reduce.mul.v8i8(<8 x i8> %ld)2343  %mul = mul i8 %a0, %rdx2344  ret i8 %mul2345}2346 2347define i8 @PR51858(i128 %arg) {2348; SSE2-LABEL: PR51858:2349; SSE2:       # %bb.0:2350; SSE2-NEXT:    movq %rdi, %xmm02351; SSE2-NEXT:    movq %rsi, %xmm12352; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]2353; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]2354; SSE2-NEXT:    pmullw %xmm1, %xmm02355; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]2356; SSE2-NEXT:    pmullw %xmm0, %xmm12357; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]2358; SSE2-NEXT:    pmullw %xmm1, %xmm02359; SSE2-NEXT:    movdqa %xmm0, %xmm12360; SSE2-NEXT:    psrld $16, %xmm12361; SSE2-NEXT:    pmullw %xmm0, %xmm12362; SSE2-NEXT:    movd %xmm1, %eax2363; SSE2-NEXT:    # kill: def $al killed $al killed $eax2364; SSE2-NEXT:    retq2365;2366; SSE41-LABEL: PR51858:2367; SSE41:       # %bb.0:2368; SSE41-NEXT:    movq %rdi, %xmm02369; SSE41-NEXT:    movq %rsi, %xmm12370; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero2371; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero2372; SSE41-NEXT:    pmullw %xmm1, %xmm02373; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]2374; SSE41-NEXT:    pmullw %xmm0, %xmm12375; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]2376; SSE41-NEXT:    pmullw %xmm1, %xmm02377; SSE41-NEXT:    movdqa %xmm0, %xmm12378; SSE41-NEXT:    psrld $16, %xmm12379; SSE41-NEXT:    pmullw %xmm0, %xmm12380; SSE41-NEXT:    movd %xmm1, %eax2381; SSE41-NEXT:    # kill: def $al killed $al killed $eax2382; SSE41-NEXT:    retq2383;2384; AVX-LABEL: PR51858:2385; AVX:       # %bb.0:2386; AVX-NEXT:    vmovq %rdi, %xmm02387; AVX-NEXT:    vmovq %rsi, %xmm12388; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero2389; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero2390; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm02391; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]2392; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm02393; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]2394; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm02395; AVX-NEXT:    vpsrld $16, %xmm0, %xmm12396; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm02397; AVX-NEXT:    vmovd %xmm0, %eax2398; AVX-NEXT:    # kill: def $al killed $al killed $eax2399; AVX-NEXT:    retq2400  %vec = bitcast i128 %arg to <16 x i8>2401  %red = tail call i8 @llvm.vector.reduce.mul.v16i8(<16 x i8> %vec)2402  ret i8 %red2403}2404 2405declare i64 @llvm.vector.reduce.mul.v2i64(<2 x i64>)2406declare i64 @llvm.vector.reduce.mul.v4i64(<4 x i64>)2407declare i64 @llvm.vector.reduce.mul.v8i64(<8 x i64>)2408declare i64 @llvm.vector.reduce.mul.v16i64(<16 x i64>)2409 2410declare i32 @llvm.vector.reduce.mul.v2i32(<2 x i32>)2411declare i32 @llvm.vector.reduce.mul.v4i32(<4 x i32>)2412declare i32 @llvm.vector.reduce.mul.v8i32(<8 x i32>)2413declare i32 @llvm.vector.reduce.mul.v16i32(<16 x i32>)2414declare i32 @llvm.vector.reduce.mul.v32i32(<32 x i32>)2415 2416declare i16 @llvm.vector.reduce.mul.v2i16(<2 x i16>)2417declare i16 @llvm.vector.reduce.mul.v4i16(<4 x i16>)2418declare i16 @llvm.vector.reduce.mul.v8i16(<8 x i16>)2419declare i16 @llvm.vector.reduce.mul.v16i16(<16 x i16>)2420declare i16 @llvm.vector.reduce.mul.v32i16(<32 x i16>)2421declare i16 @llvm.vector.reduce.mul.v64i16(<64 x i16>)2422 2423declare i8 @llvm.vector.reduce.mul.v2i8(<2 x i8>)2424declare i8 @llvm.vector.reduce.mul.v4i8(<4 x i8>)2425declare i8 @llvm.vector.reduce.mul.v8i8(<8 x i8>)2426declare i8 @llvm.vector.reduce.mul.v16i8(<16 x i8>)2427declare i8 @llvm.vector.reduce.mul.v32i8(<32 x i8>)2428declare i8 @llvm.vector.reduce.mul.v64i8(<64 x i8>)2429declare i8 @llvm.vector.reduce.mul.v128i8(<128 x i8>)2430