1349 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=SSE --check-prefix=SSE23; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE414; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX25; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW7 8define <16 x i8> @mul_v16i8c(<16 x i8> %i) nounwind {9; SSE2-LABEL: mul_v16i8c:10; SSE2: # %bb.0: # %entry11; SSE2-NEXT: movdqa %xmm0, %xmm112; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]13; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [117,u,117,u,117,u,117,u,117,u,117,u,117,u,117,u]14; SSE2-NEXT: pmullw %xmm2, %xmm115; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]16; SSE2-NEXT: pand %xmm3, %xmm117; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]18; SSE2-NEXT: pmullw %xmm2, %xmm019; SSE2-NEXT: pand %xmm3, %xmm020; SSE2-NEXT: packuswb %xmm1, %xmm021; SSE2-NEXT: retq22;23; SSE41-LABEL: mul_v16i8c:24; SSE41: # %bb.0: # %entry25; SSE41-NEXT: movdqa %xmm0, %xmm126; SSE41-NEXT: pmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 # [0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117]27; SSE41-NEXT: psllw $8, %xmm128; SSE41-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]29; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm030; SSE41-NEXT: por %xmm1, %xmm031; SSE41-NEXT: retq32;33; AVX2-LABEL: mul_v16i8c:34; AVX2: # %bb.0: # %entry35; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero36; AVX2-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]37; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm038; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm139; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm040; AVX2-NEXT: vzeroupper41; AVX2-NEXT: retq42;43; AVX512F-LABEL: mul_v16i8c:44; AVX512F: # %bb.0: # %entry45; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero46; AVX512F-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]47; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero48; AVX512F-NEXT: vpmovdb %zmm0, %xmm049; AVX512F-NEXT: vzeroupper50; AVX512F-NEXT: retq51;52; AVX512BW-LABEL: mul_v16i8c:53; AVX512BW: # %bb.0: # %entry54; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero55; AVX512BW-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]56; AVX512BW-NEXT: vpmovwb %zmm0, %ymm057; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm058; AVX512BW-NEXT: vzeroupper59; AVX512BW-NEXT: retq60entry:61 %A = mul <16 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 >62 ret <16 x i8> %A63}64 65define <8 x i16> @mul_v8i16c(<8 x i16> %i) nounwind {66; SSE-LABEL: mul_v8i16c:67; SSE: # %bb.0: # %entry68; SSE-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [117,117,117,117,117,117,117,117]69; SSE-NEXT: retq70;71; AVX-LABEL: mul_v8i16c:72; AVX: # %bb.0: # %entry73; AVX-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [117,117,117,117,117,117,117,117]74; AVX-NEXT: retq75entry:76 %A = mul <8 x i16> %i, < i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117 >77 ret <8 x i16> %A78}79 80define <4 x i32> @mul_v4i32c(<4 x i32> %i) nounwind {81; SSE2-LABEL: mul_v4i32c:82; SSE2: # %bb.0: # %entry83; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [117,117,117,117]84; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]85; SSE2-NEXT: pmuludq %xmm1, %xmm086; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]87; SSE2-NEXT: pmuludq %xmm1, %xmm288; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]89; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]90; SSE2-NEXT: retq91;92; SSE41-LABEL: mul_v4i32c:93; SSE41: # %bb.0: # %entry94; SSE41-NEXT: pmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [117,117,117,117]95; SSE41-NEXT: retq96;97; AVX-LABEL: mul_v4i32c:98; AVX: # %bb.0: # %entry99; AVX-NEXT: vpbroadcastd {{.*#+}} xmm1 = [117,117,117,117]100; AVX-NEXT: vpmulld %xmm1, %xmm0, %xmm0101; AVX-NEXT: retq102entry:103 %A = mul <4 x i32> %i, < i32 117, i32 117, i32 117, i32 117 >104 ret <4 x i32> %A105}106 107define <2 x i64> @mul_v2i64c(<2 x i64> %i) nounwind {108; SSE2-LABEL: mul_v2i64c:109; SSE2: # %bb.0: # %entry110; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [117,117]111; SSE2-NEXT: movdqa %xmm0, %xmm2112; SSE2-NEXT: pmuludq %xmm1, %xmm2113; SSE2-NEXT: psrlq $32, %xmm0114; SSE2-NEXT: pmuludq %xmm1, %xmm0115; SSE2-NEXT: psllq $32, %xmm0116; SSE2-NEXT: paddq %xmm2, %xmm0117; SSE2-NEXT: retq118;119; SSE41-LABEL: mul_v2i64c:120; SSE41: # %bb.0: # %entry121; SSE41-NEXT: pmovsxbq {{.*#+}} xmm1 = [117,117]122; SSE41-NEXT: movdqa %xmm0, %xmm2123; SSE41-NEXT: pmuludq %xmm1, %xmm2124; SSE41-NEXT: psrlq $32, %xmm0125; SSE41-NEXT: pmuludq %xmm1, %xmm0126; SSE41-NEXT: psllq $32, %xmm0127; SSE41-NEXT: paddq %xmm2, %xmm0128; SSE41-NEXT: retq129;130; AVX-LABEL: mul_v2i64c:131; AVX: # %bb.0: # %entry132; AVX-NEXT: vpmovsxbq {{.*#+}} xmm1 = [117,117]133; AVX-NEXT: vpmuludq %xmm1, %xmm0, %xmm2134; AVX-NEXT: vpsrlq $32, %xmm0, %xmm0135; AVX-NEXT: vpmuludq %xmm1, %xmm0, %xmm0136; AVX-NEXT: vpsllq $32, %xmm0, %xmm0137; AVX-NEXT: vpaddq %xmm0, %xmm2, %xmm0138; AVX-NEXT: retq139entry:140 %A = mul <2 x i64> %i, < i64 117, i64 117 >141 ret <2 x i64> %A142}143 144define <16 x i8> @mul_v16i8(<16 x i8> %i, <16 x i8> %j) nounwind {145; SSE2-LABEL: mul_v16i8:146; SSE2: # %bb.0: # %entry147; SSE2-NEXT: movdqa %xmm1, %xmm2148; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]149; SSE2-NEXT: movdqa %xmm0, %xmm3150; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]151; SSE2-NEXT: pmullw %xmm2, %xmm3152; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]153; SSE2-NEXT: pand %xmm2, %xmm3154; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]155; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]156; SSE2-NEXT: pmullw %xmm1, %xmm0157; SSE2-NEXT: pand %xmm2, %xmm0158; SSE2-NEXT: packuswb %xmm3, %xmm0159; SSE2-NEXT: retq160;161; SSE41-LABEL: mul_v16i8:162; SSE41: # %bb.0: # %entry163; SSE41-NEXT: movdqa %xmm0, %xmm2164; SSE41-NEXT: pmullw %xmm1, %xmm2165; SSE41-NEXT: pmovzxbw {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]166; SSE41-NEXT: pand %xmm3, %xmm2167; SSE41-NEXT: pandn %xmm1, %xmm3168; SSE41-NEXT: pmaddubsw %xmm3, %xmm0169; SSE41-NEXT: psllw $8, %xmm0170; SSE41-NEXT: por %xmm2, %xmm0171; SSE41-NEXT: retq172;173; AVX2-LABEL: mul_v16i8:174; AVX2: # %bb.0: # %entry175; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero176; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero177; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm0178; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0179; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1180; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0181; AVX2-NEXT: vzeroupper182; AVX2-NEXT: retq183;184; AVX512F-LABEL: mul_v16i8:185; AVX512F: # %bb.0: # %entry186; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero187; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero188; AVX512F-NEXT: vpmullw %ymm1, %ymm0, %ymm0189; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero190; AVX512F-NEXT: vpmovdb %zmm0, %xmm0191; AVX512F-NEXT: vzeroupper192; AVX512F-NEXT: retq193;194; AVX512BW-LABEL: mul_v16i8:195; AVX512BW: # %bb.0: # %entry196; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero197; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero198; AVX512BW-NEXT: vpmullw %ymm1, %ymm0, %ymm0199; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0200; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0201; AVX512BW-NEXT: vzeroupper202; AVX512BW-NEXT: retq203entry:204 %A = mul <16 x i8> %i, %j205 ret <16 x i8> %A206}207 208define <8 x i16> @mul_v8i16(<8 x i16> %i, <8 x i16> %j) nounwind {209; SSE-LABEL: mul_v8i16:210; SSE: # %bb.0: # %entry211; SSE-NEXT: pmullw %xmm1, %xmm0212; SSE-NEXT: retq213;214; AVX-LABEL: mul_v8i16:215; AVX: # %bb.0: # %entry216; AVX-NEXT: vpmullw %xmm1, %xmm0, %xmm0217; AVX-NEXT: retq218entry:219 %A = mul <8 x i16> %i, %j220 ret <8 x i16> %A221}222 223define <4 x i32> @mul_v4i32(<4 x i32> %i, <4 x i32> %j) nounwind {224; SSE2-LABEL: mul_v4i32:225; SSE2: # %bb.0: # %entry226; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]227; SSE2-NEXT: pmuludq %xmm1, %xmm0228; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]229; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]230; SSE2-NEXT: pmuludq %xmm2, %xmm1231; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]232; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]233; SSE2-NEXT: retq234;235; SSE41-LABEL: mul_v4i32:236; SSE41: # %bb.0: # %entry237; SSE41-NEXT: pmulld %xmm1, %xmm0238; SSE41-NEXT: retq239;240; AVX-LABEL: mul_v4i32:241; AVX: # %bb.0: # %entry242; AVX-NEXT: vpmulld %xmm1, %xmm0, %xmm0243; AVX-NEXT: retq244entry:245 %A = mul <4 x i32> %i, %j246 ret <4 x i32> %A247}248 249define <2 x i64> @mul_v2i64(<2 x i64> %i, <2 x i64> %j) nounwind {250; SSE-LABEL: mul_v2i64:251; SSE: # %bb.0: # %entry252; SSE-NEXT: movdqa %xmm0, %xmm2253; SSE-NEXT: psrlq $32, %xmm2254; SSE-NEXT: pmuludq %xmm1, %xmm2255; SSE-NEXT: movdqa %xmm1, %xmm3256; SSE-NEXT: psrlq $32, %xmm3257; SSE-NEXT: pmuludq %xmm0, %xmm3258; SSE-NEXT: paddq %xmm2, %xmm3259; SSE-NEXT: psllq $32, %xmm3260; SSE-NEXT: pmuludq %xmm1, %xmm0261; SSE-NEXT: paddq %xmm3, %xmm0262; SSE-NEXT: retq263;264; AVX-LABEL: mul_v2i64:265; AVX: # %bb.0: # %entry266; AVX-NEXT: vpsrlq $32, %xmm0, %xmm2267; AVX-NEXT: vpmuludq %xmm1, %xmm2, %xmm2268; AVX-NEXT: vpsrlq $32, %xmm1, %xmm3269; AVX-NEXT: vpmuludq %xmm3, %xmm0, %xmm3270; AVX-NEXT: vpaddq %xmm2, %xmm3, %xmm2271; AVX-NEXT: vpsllq $32, %xmm2, %xmm2272; AVX-NEXT: vpmuludq %xmm1, %xmm0, %xmm0273; AVX-NEXT: vpaddq %xmm2, %xmm0, %xmm0274; AVX-NEXT: retq275entry:276 %A = mul <2 x i64> %i, %j277 ret <2 x i64> %A278}279 280declare void @foo()281 282define <4 x i32> @mul_v4i32spill(<4 x i32> %i, <4 x i32> %j) nounwind {283; SSE2-LABEL: mul_v4i32spill:284; SSE2: # %bb.0: # %entry285; SSE2-NEXT: subq $40, %rsp286; SSE2-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill287; SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill288; SSE2-NEXT: callq foo@PLT289; SSE2-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload290; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]291; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload292; SSE2-NEXT: pmuludq %xmm2, %xmm0293; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]294; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]295; SSE2-NEXT: pmuludq %xmm1, %xmm2296; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]297; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]298; SSE2-NEXT: addq $40, %rsp299; SSE2-NEXT: retq300;301; SSE41-LABEL: mul_v4i32spill:302; SSE41: # %bb.0: # %entry303; SSE41-NEXT: subq $40, %rsp304; SSE41-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill305; SSE41-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill306; SSE41-NEXT: callq foo@PLT307; SSE41-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload308; SSE41-NEXT: pmulld {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload309; SSE41-NEXT: addq $40, %rsp310; SSE41-NEXT: retq311;312; AVX-LABEL: mul_v4i32spill:313; AVX: # %bb.0: # %entry314; AVX-NEXT: subq $40, %rsp315; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill316; AVX-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill317; AVX-NEXT: callq foo@PLT318; AVX-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload319; AVX-NEXT: vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload320; AVX-NEXT: addq $40, %rsp321; AVX-NEXT: retq322entry:323 ; Use a call to force spills.324 call void @foo()325 %A = mul <4 x i32> %i, %j326 ret <4 x i32> %A327}328 329define <2 x i64> @mul_v2i64spill(<2 x i64> %i, <2 x i64> %j) nounwind {330; SSE-LABEL: mul_v2i64spill:331; SSE: # %bb.0: # %entry332; SSE-NEXT: subq $40, %rsp333; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill334; SSE-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill335; SSE-NEXT: callq foo@PLT336; SSE-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload337; SSE-NEXT: movdqa %xmm0, %xmm2338; SSE-NEXT: psrlq $32, %xmm2339; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload340; SSE-NEXT: pmuludq %xmm3, %xmm2341; SSE-NEXT: movdqa %xmm3, %xmm1342; SSE-NEXT: psrlq $32, %xmm1343; SSE-NEXT: pmuludq %xmm0, %xmm1344; SSE-NEXT: paddq %xmm2, %xmm1345; SSE-NEXT: psllq $32, %xmm1346; SSE-NEXT: pmuludq %xmm3, %xmm0347; SSE-NEXT: paddq %xmm1, %xmm0348; SSE-NEXT: addq $40, %rsp349; SSE-NEXT: retq350;351; AVX-LABEL: mul_v2i64spill:352; AVX: # %bb.0: # %entry353; AVX-NEXT: subq $40, %rsp354; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill355; AVX-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill356; AVX-NEXT: callq foo@PLT357; AVX-NEXT: vmovdqa (%rsp), %xmm3 # 16-byte Reload358; AVX-NEXT: vpsrlq $32, %xmm3, %xmm0359; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload360; AVX-NEXT: vpmuludq %xmm2, %xmm0, %xmm0361; AVX-NEXT: vpsrlq $32, %xmm2, %xmm1362; AVX-NEXT: vpmuludq %xmm1, %xmm3, %xmm1363; AVX-NEXT: vpaddq %xmm0, %xmm1, %xmm0364; AVX-NEXT: vpsllq $32, %xmm0, %xmm0365; AVX-NEXT: vpmuludq %xmm2, %xmm3, %xmm1366; AVX-NEXT: vpaddq %xmm0, %xmm1, %xmm0367; AVX-NEXT: addq $40, %rsp368; AVX-NEXT: retq369entry:370 ; Use a call to force spills.371 call void @foo()372 %A = mul <2 x i64> %i, %j373 ret <2 x i64> %A374}375 376define <32 x i8> @mul_v32i8c(<32 x i8> %i) nounwind {377; SSE2-LABEL: mul_v32i8c:378; SSE2: # %bb.0: # %entry379; SSE2-NEXT: movdqa %xmm0, %xmm2380; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]381; SSE2-NEXT: movdqa {{.*#+}} xmm3 = [117,u,117,u,117,u,117,u,117,u,117,u,117,u,117,u]382; SSE2-NEXT: pmullw %xmm3, %xmm2383; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]384; SSE2-NEXT: pand %xmm4, %xmm2385; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]386; SSE2-NEXT: pmullw %xmm3, %xmm0387; SSE2-NEXT: pand %xmm4, %xmm0388; SSE2-NEXT: packuswb %xmm2, %xmm0389; SSE2-NEXT: movdqa %xmm1, %xmm2390; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]391; SSE2-NEXT: pmullw %xmm3, %xmm2392; SSE2-NEXT: pand %xmm4, %xmm2393; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]394; SSE2-NEXT: pmullw %xmm3, %xmm1395; SSE2-NEXT: pand %xmm4, %xmm1396; SSE2-NEXT: packuswb %xmm2, %xmm1397; SSE2-NEXT: retq398;399; SSE41-LABEL: mul_v32i8c:400; SSE41: # %bb.0: # %entry401; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]402; SSE41-NEXT: movdqa %xmm0, %xmm3403; SSE41-NEXT: pmullw %xmm2, %xmm3404; SSE41-NEXT: pmovzxbw {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]405; SSE41-NEXT: pand %xmm4, %xmm3406; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117]407; SSE41-NEXT: pmaddubsw %xmm5, %xmm0408; SSE41-NEXT: psllw $8, %xmm0409; SSE41-NEXT: por %xmm3, %xmm0410; SSE41-NEXT: pmullw %xmm1, %xmm2411; SSE41-NEXT: pand %xmm4, %xmm2412; SSE41-NEXT: pmaddubsw %xmm5, %xmm1413; SSE41-NEXT: psllw $8, %xmm1414; SSE41-NEXT: por %xmm2, %xmm1415; SSE41-NEXT: retq416;417; AVX2-LABEL: mul_v32i8c:418; AVX2: # %bb.0: # %entry419; AVX2-NEXT: vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1 # [0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117]420; AVX2-NEXT: vpsllw $8, %ymm1, %ymm1421; AVX2-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]422; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0423; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0424; AVX2-NEXT: retq425;426; AVX512F-LABEL: mul_v32i8c:427; AVX512F: # %bb.0: # %entry428; AVX512F-NEXT: vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1 # [0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117]429; AVX512F-NEXT: vpsllw $8, %ymm1, %ymm1430; AVX512F-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]431; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0432; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0433; AVX512F-NEXT: retq434;435; AVX512BW-LABEL: mul_v32i8c:436; AVX512BW: # %bb.0: # %entry437; AVX512BW-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero438; AVX512BW-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]439; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0440; AVX512BW-NEXT: retq441entry:442 %A = mul <32 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 >443 ret <32 x i8> %A444}445 446define <16 x i16> @mul_v16i16c(<16 x i16> %i) nounwind {447; SSE2-LABEL: mul_v16i16c:448; SSE2: # %bb.0: # %entry449; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117]450; SSE2-NEXT: pmullw %xmm2, %xmm0451; SSE2-NEXT: pmullw %xmm2, %xmm1452; SSE2-NEXT: retq453;454; SSE41-LABEL: mul_v16i16c:455; SSE41: # %bb.0: # %entry456; SSE41-NEXT: pmovsxbw {{.*#+}} xmm2 = [117,117,117,117,117,117,117,117]457; SSE41-NEXT: pmullw %xmm2, %xmm0458; SSE41-NEXT: pmullw %xmm2, %xmm1459; SSE41-NEXT: retq460;461; AVX-LABEL: mul_v16i16c:462; AVX: # %bb.0: # %entry463; AVX-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]464; AVX-NEXT: retq465entry:466 %A = mul <16 x i16> %i, < i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117, i16 117 >467 ret <16 x i16> %A468}469 470define <8 x i32> @mul_v8i32c(<8 x i32> %i) nounwind {471; SSE2-LABEL: mul_v8i32c:472; SSE2: # %bb.0: # %entry473; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [117,117,117,117]474; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]475; SSE2-NEXT: pmuludq %xmm2, %xmm0476; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]477; SSE2-NEXT: pmuludq %xmm2, %xmm3478; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]479; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]480; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]481; SSE2-NEXT: pmuludq %xmm2, %xmm1482; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]483; SSE2-NEXT: pmuludq %xmm2, %xmm3484; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]485; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]486; SSE2-NEXT: retq487;488; SSE41-LABEL: mul_v8i32c:489; SSE41: # %bb.0: # %entry490; SSE41-NEXT: pmovsxbd {{.*#+}} xmm2 = [117,117,117,117]491; SSE41-NEXT: pmulld %xmm2, %xmm0492; SSE41-NEXT: pmulld %xmm2, %xmm1493; SSE41-NEXT: retq494;495; AVX-LABEL: mul_v8i32c:496; AVX: # %bb.0: # %entry497; AVX-NEXT: vpbroadcastd {{.*#+}} ymm1 = [117,117,117,117,117,117,117,117]498; AVX-NEXT: vpmulld %ymm1, %ymm0, %ymm0499; AVX-NEXT: retq500entry:501 %A = mul <8 x i32> %i, < i32 117, i32 117, i32 117, i32 117, i32 117, i32 117, i32 117, i32 117 >502 ret <8 x i32> %A503}504 505define <4 x i64> @mul_v4i64c(<4 x i64> %i) nounwind {506; SSE2-LABEL: mul_v4i64c:507; SSE2: # %bb.0: # %entry508; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [117,117]509; SSE2-NEXT: movdqa %xmm0, %xmm3510; SSE2-NEXT: pmuludq %xmm2, %xmm3511; SSE2-NEXT: psrlq $32, %xmm0512; SSE2-NEXT: pmuludq %xmm2, %xmm0513; SSE2-NEXT: psllq $32, %xmm0514; SSE2-NEXT: paddq %xmm3, %xmm0515; SSE2-NEXT: movdqa %xmm1, %xmm3516; SSE2-NEXT: pmuludq %xmm2, %xmm3517; SSE2-NEXT: psrlq $32, %xmm1518; SSE2-NEXT: pmuludq %xmm2, %xmm1519; SSE2-NEXT: psllq $32, %xmm1520; SSE2-NEXT: paddq %xmm3, %xmm1521; SSE2-NEXT: retq522;523; SSE41-LABEL: mul_v4i64c:524; SSE41: # %bb.0: # %entry525; SSE41-NEXT: pmovsxbq {{.*#+}} xmm2 = [117,117]526; SSE41-NEXT: movdqa %xmm0, %xmm3527; SSE41-NEXT: pmuludq %xmm2, %xmm3528; SSE41-NEXT: psrlq $32, %xmm0529; SSE41-NEXT: pmuludq %xmm2, %xmm0530; SSE41-NEXT: psllq $32, %xmm0531; SSE41-NEXT: paddq %xmm3, %xmm0532; SSE41-NEXT: movdqa %xmm1, %xmm3533; SSE41-NEXT: pmuludq %xmm2, %xmm3534; SSE41-NEXT: psrlq $32, %xmm1535; SSE41-NEXT: pmuludq %xmm2, %xmm1536; SSE41-NEXT: psllq $32, %xmm1537; SSE41-NEXT: paddq %xmm3, %xmm1538; SSE41-NEXT: retq539;540; AVX-LABEL: mul_v4i64c:541; AVX: # %bb.0: # %entry542; AVX-NEXT: vpbroadcastq {{.*#+}} ymm1 = [117,117,117,117]543; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm2544; AVX-NEXT: vpsrlq $32, %ymm0, %ymm0545; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm0546; AVX-NEXT: vpsllq $32, %ymm0, %ymm0547; AVX-NEXT: vpaddq %ymm0, %ymm2, %ymm0548; AVX-NEXT: retq549entry:550 %A = mul <4 x i64> %i, < i64 117, i64 117, i64 117, i64 117 >551 ret <4 x i64> %A552}553 554define <32 x i8> @mul_v32i8(<32 x i8> %i, <32 x i8> %j) nounwind {555; SSE2-LABEL: mul_v32i8:556; SSE2: # %bb.0: # %entry557; SSE2-NEXT: movdqa %xmm2, %xmm4558; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]559; SSE2-NEXT: movdqa %xmm0, %xmm5560; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]561; SSE2-NEXT: pmullw %xmm4, %xmm5562; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]563; SSE2-NEXT: pand %xmm4, %xmm5564; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]565; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]566; SSE2-NEXT: pmullw %xmm2, %xmm0567; SSE2-NEXT: pand %xmm4, %xmm0568; SSE2-NEXT: packuswb %xmm5, %xmm0569; SSE2-NEXT: movdqa %xmm3, %xmm2570; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]571; SSE2-NEXT: movdqa %xmm1, %xmm5572; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]573; SSE2-NEXT: pmullw %xmm2, %xmm5574; SSE2-NEXT: pand %xmm4, %xmm5575; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]576; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]577; SSE2-NEXT: pmullw %xmm3, %xmm1578; SSE2-NEXT: pand %xmm4, %xmm1579; SSE2-NEXT: packuswb %xmm5, %xmm1580; SSE2-NEXT: retq581;582; SSE41-LABEL: mul_v32i8:583; SSE41: # %bb.0: # %entry584; SSE41-NEXT: movdqa %xmm0, %xmm4585; SSE41-NEXT: pmullw %xmm2, %xmm4586; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]587; SSE41-NEXT: pand %xmm5, %xmm4588; SSE41-NEXT: movdqa %xmm5, %xmm6589; SSE41-NEXT: pandn %xmm2, %xmm6590; SSE41-NEXT: pmaddubsw %xmm6, %xmm0591; SSE41-NEXT: psllw $8, %xmm0592; SSE41-NEXT: por %xmm4, %xmm0593; SSE41-NEXT: movdqa %xmm1, %xmm2594; SSE41-NEXT: pmullw %xmm3, %xmm2595; SSE41-NEXT: pand %xmm5, %xmm2596; SSE41-NEXT: pandn %xmm3, %xmm5597; SSE41-NEXT: pmaddubsw %xmm5, %xmm1598; SSE41-NEXT: psllw $8, %xmm1599; SSE41-NEXT: por %xmm2, %xmm1600; SSE41-NEXT: retq601;602; AVX2-LABEL: mul_v32i8:603; AVX2: # %bb.0: # %entry604; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm2605; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]606; AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2607; AVX2-NEXT: vpandn %ymm1, %ymm3, %ymm1608; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0609; AVX2-NEXT: vpsllw $8, %ymm0, %ymm0610; AVX2-NEXT: vpor %ymm0, %ymm2, %ymm0611; AVX2-NEXT: retq612;613; AVX512F-LABEL: mul_v32i8:614; AVX512F: # %bb.0: # %entry615; AVX512F-NEXT: vpmullw %ymm1, %ymm0, %ymm2616; AVX512F-NEXT: vpbroadcastw {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]617; AVX512F-NEXT: vpand %ymm3, %ymm2, %ymm2618; AVX512F-NEXT: vpandn %ymm1, %ymm3, %ymm1619; AVX512F-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0620; AVX512F-NEXT: vpsllw $8, %ymm0, %ymm0621; AVX512F-NEXT: vpor %ymm0, %ymm2, %ymm0622; AVX512F-NEXT: retq623;624; AVX512BW-LABEL: mul_v32i8:625; AVX512BW: # %bb.0: # %entry626; AVX512BW-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero627; AVX512BW-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero628; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0629; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0630; AVX512BW-NEXT: retq631entry:632 %A = mul <32 x i8> %i, %j633 ret <32 x i8> %A634}635 636define <16 x i16> @mul_v16i16(<16 x i16> %i, <16 x i16> %j) nounwind {637; SSE-LABEL: mul_v16i16:638; SSE: # %bb.0: # %entry639; SSE-NEXT: pmullw %xmm2, %xmm0640; SSE-NEXT: pmullw %xmm3, %xmm1641; SSE-NEXT: retq642;643; AVX-LABEL: mul_v16i16:644; AVX: # %bb.0: # %entry645; AVX-NEXT: vpmullw %ymm1, %ymm0, %ymm0646; AVX-NEXT: retq647entry:648 %A = mul <16 x i16> %i, %j649 ret <16 x i16> %A650}651 652define <8 x i32> @mul_v8i32(<8 x i32> %i, <8 x i32> %j) nounwind {653; SSE2-LABEL: mul_v8i32:654; SSE2: # %bb.0: # %entry655; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]656; SSE2-NEXT: pmuludq %xmm2, %xmm0657; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]658; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]659; SSE2-NEXT: pmuludq %xmm4, %xmm2660; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]661; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]662; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]663; SSE2-NEXT: pmuludq %xmm3, %xmm1664; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]665; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]666; SSE2-NEXT: pmuludq %xmm2, %xmm3667; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]668; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]669; SSE2-NEXT: retq670;671; SSE41-LABEL: mul_v8i32:672; SSE41: # %bb.0: # %entry673; SSE41-NEXT: pmulld %xmm2, %xmm0674; SSE41-NEXT: pmulld %xmm3, %xmm1675; SSE41-NEXT: retq676;677; AVX-LABEL: mul_v8i32:678; AVX: # %bb.0: # %entry679; AVX-NEXT: vpmulld %ymm1, %ymm0, %ymm0680; AVX-NEXT: retq681entry:682 %A = mul <8 x i32> %i, %j683 ret <8 x i32> %A684}685 686define <4 x i64> @mul_v4i64(<4 x i64> %i, <4 x i64> %j) nounwind {687; SSE-LABEL: mul_v4i64:688; SSE: # %bb.0: # %entry689; SSE-NEXT: movdqa %xmm0, %xmm4690; SSE-NEXT: psrlq $32, %xmm4691; SSE-NEXT: pmuludq %xmm2, %xmm4692; SSE-NEXT: movdqa %xmm2, %xmm5693; SSE-NEXT: psrlq $32, %xmm5694; SSE-NEXT: pmuludq %xmm0, %xmm5695; SSE-NEXT: paddq %xmm4, %xmm5696; SSE-NEXT: psllq $32, %xmm5697; SSE-NEXT: pmuludq %xmm2, %xmm0698; SSE-NEXT: paddq %xmm5, %xmm0699; SSE-NEXT: movdqa %xmm1, %xmm2700; SSE-NEXT: psrlq $32, %xmm2701; SSE-NEXT: pmuludq %xmm3, %xmm2702; SSE-NEXT: movdqa %xmm3, %xmm4703; SSE-NEXT: psrlq $32, %xmm4704; SSE-NEXT: pmuludq %xmm1, %xmm4705; SSE-NEXT: paddq %xmm2, %xmm4706; SSE-NEXT: psllq $32, %xmm4707; SSE-NEXT: pmuludq %xmm3, %xmm1708; SSE-NEXT: paddq %xmm4, %xmm1709; SSE-NEXT: retq710;711; AVX-LABEL: mul_v4i64:712; AVX: # %bb.0: # %entry713; AVX-NEXT: vpsrlq $32, %ymm0, %ymm2714; AVX-NEXT: vpmuludq %ymm1, %ymm2, %ymm2715; AVX-NEXT: vpsrlq $32, %ymm1, %ymm3716; AVX-NEXT: vpmuludq %ymm3, %ymm0, %ymm3717; AVX-NEXT: vpaddq %ymm2, %ymm3, %ymm2718; AVX-NEXT: vpsllq $32, %ymm2, %ymm2719; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm0720; AVX-NEXT: vpaddq %ymm2, %ymm0, %ymm0721; AVX-NEXT: retq722entry:723 %A = mul <4 x i64> %i, %j724 ret <4 x i64> %A725}726 727define <64 x i8> @mul_v64i8c(<64 x i8> %i) nounwind {728; SSE2-LABEL: mul_v64i8c:729; SSE2: # %bb.0: # %entry730; SSE2-NEXT: movdqa %xmm0, %xmm6731; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]732; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [117,u,117,u,117,u,117,u,117,u,117,u,117,u,117,u]733; SSE2-NEXT: pmullw %xmm4, %xmm6734; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]735; SSE2-NEXT: pand %xmm5, %xmm6736; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]737; SSE2-NEXT: pmullw %xmm4, %xmm0738; SSE2-NEXT: pand %xmm5, %xmm0739; SSE2-NEXT: packuswb %xmm6, %xmm0740; SSE2-NEXT: movdqa %xmm1, %xmm6741; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]742; SSE2-NEXT: pmullw %xmm4, %xmm6743; SSE2-NEXT: pand %xmm5, %xmm6744; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]745; SSE2-NEXT: pmullw %xmm4, %xmm1746; SSE2-NEXT: pand %xmm5, %xmm1747; SSE2-NEXT: packuswb %xmm6, %xmm1748; SSE2-NEXT: movdqa %xmm2, %xmm6749; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]750; SSE2-NEXT: pmullw %xmm4, %xmm6751; SSE2-NEXT: pand %xmm5, %xmm6752; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]753; SSE2-NEXT: pmullw %xmm4, %xmm2754; SSE2-NEXT: pand %xmm5, %xmm2755; SSE2-NEXT: packuswb %xmm6, %xmm2756; SSE2-NEXT: movdqa %xmm3, %xmm6757; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]758; SSE2-NEXT: pmullw %xmm4, %xmm6759; SSE2-NEXT: pand %xmm5, %xmm6760; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]761; SSE2-NEXT: pmullw %xmm4, %xmm3762; SSE2-NEXT: pand %xmm5, %xmm3763; SSE2-NEXT: packuswb %xmm6, %xmm3764; SSE2-NEXT: retq765;766; SSE41-LABEL: mul_v64i8c:767; SSE41: # %bb.0: # %entry768; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]769; SSE41-NEXT: movdqa %xmm0, %xmm6770; SSE41-NEXT: pmullw %xmm4, %xmm6771; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]772; SSE41-NEXT: pand %xmm5, %xmm6773; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117]774; SSE41-NEXT: pmaddubsw %xmm7, %xmm0775; SSE41-NEXT: psllw $8, %xmm0776; SSE41-NEXT: por %xmm6, %xmm0777; SSE41-NEXT: movdqa %xmm1, %xmm6778; SSE41-NEXT: pmullw %xmm4, %xmm6779; SSE41-NEXT: pand %xmm5, %xmm6780; SSE41-NEXT: pmaddubsw %xmm7, %xmm1781; SSE41-NEXT: psllw $8, %xmm1782; SSE41-NEXT: por %xmm6, %xmm1783; SSE41-NEXT: movdqa %xmm2, %xmm6784; SSE41-NEXT: pmullw %xmm4, %xmm6785; SSE41-NEXT: pand %xmm5, %xmm6786; SSE41-NEXT: pmaddubsw %xmm7, %xmm2787; SSE41-NEXT: psllw $8, %xmm2788; SSE41-NEXT: por %xmm6, %xmm2789; SSE41-NEXT: pmullw %xmm3, %xmm4790; SSE41-NEXT: pand %xmm5, %xmm4791; SSE41-NEXT: pmaddubsw %xmm7, %xmm3792; SSE41-NEXT: psllw $8, %xmm3793; SSE41-NEXT: por %xmm4, %xmm3794; SSE41-NEXT: retq795;796; AVX2-LABEL: mul_v64i8c:797; AVX2: # %bb.0: # %entry798; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]799; AVX2-NEXT: vpmullw %ymm2, %ymm0, %ymm3800; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]801; AVX2-NEXT: vpand %ymm4, %ymm3, %ymm3802; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm5 = [0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117]803; AVX2-NEXT: vpmaddubsw %ymm5, %ymm0, %ymm0804; AVX2-NEXT: vpsllw $8, %ymm0, %ymm0805; AVX2-NEXT: vpor %ymm0, %ymm3, %ymm0806; AVX2-NEXT: vpmullw %ymm2, %ymm1, %ymm2807; AVX2-NEXT: vpand %ymm4, %ymm2, %ymm2808; AVX2-NEXT: vpmaddubsw %ymm5, %ymm1, %ymm1809; AVX2-NEXT: vpsllw $8, %ymm1, %ymm1810; AVX2-NEXT: vpor %ymm1, %ymm2, %ymm1811; AVX2-NEXT: retq812;813; AVX512F-LABEL: mul_v64i8c:814; AVX512F: # %bb.0: # %entry815; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1816; AVX512F-NEXT: vpbroadcastb {{.*#+}} ymm2 = [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]817; AVX512F-NEXT: vpmullw %ymm2, %ymm1, %ymm3818; AVX512F-NEXT: vpmullw %ymm2, %ymm0, %ymm2819; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2820; AVX512F-NEXT: vpbroadcastw {{.*#+}} ymm3 = [0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117]821; AVX512F-NEXT: vpmaddubsw %ymm3, %ymm0, %ymm0822; AVX512F-NEXT: vpsllw $8, %ymm0, %ymm0823; AVX512F-NEXT: vpmaddubsw %ymm3, %ymm1, %ymm1824; AVX512F-NEXT: vpsllw $8, %ymm1, %ymm1825; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0826; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm2 & m32bcst)827; AVX512F-NEXT: retq828;829; AVX512BW-LABEL: mul_v64i8c:830; AVX512BW: # %bb.0: # %entry831; AVX512BW-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 # [117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117,117]832; AVX512BW-NEXT: vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117,0,117]833; AVX512BW-NEXT: vpsllw $8, %zmm0, %zmm0834; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm0 = zmm0 | (zmm1 & m32bcst)835; AVX512BW-NEXT: retq836entry:837 %A = mul <64 x i8> %i, < i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117, i8 117 >838 ret <64 x i8> %A839}840 841define <64 x i8> @mul_v64i8(<64 x i8> %i, <64 x i8> %j) nounwind {842; SSE2-LABEL: mul_v64i8:843; SSE2: # %bb.0: # %entry844; SSE2-NEXT: movdqa %xmm4, %xmm8845; SSE2-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]846; SSE2-NEXT: movdqa %xmm0, %xmm9847; SSE2-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]848; SSE2-NEXT: pmullw %xmm8, %xmm9849; SSE2-NEXT: movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]850; SSE2-NEXT: pand %xmm8, %xmm9851; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]852; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]853; SSE2-NEXT: pmullw %xmm4, %xmm0854; SSE2-NEXT: pand %xmm8, %xmm0855; SSE2-NEXT: packuswb %xmm9, %xmm0856; SSE2-NEXT: movdqa %xmm5, %xmm4857; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]858; SSE2-NEXT: movdqa %xmm1, %xmm9859; SSE2-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]860; SSE2-NEXT: pmullw %xmm4, %xmm9861; SSE2-NEXT: pand %xmm8, %xmm9862; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]863; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]864; SSE2-NEXT: pmullw %xmm5, %xmm1865; SSE2-NEXT: pand %xmm8, %xmm1866; SSE2-NEXT: packuswb %xmm9, %xmm1867; SSE2-NEXT: movdqa %xmm6, %xmm4868; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]869; SSE2-NEXT: movdqa %xmm2, %xmm5870; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]871; SSE2-NEXT: pmullw %xmm4, %xmm5872; SSE2-NEXT: pand %xmm8, %xmm5873; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]874; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]875; SSE2-NEXT: pmullw %xmm6, %xmm2876; SSE2-NEXT: pand %xmm8, %xmm2877; SSE2-NEXT: packuswb %xmm5, %xmm2878; SSE2-NEXT: movdqa %xmm7, %xmm4879; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]880; SSE2-NEXT: movdqa %xmm3, %xmm5881; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]882; SSE2-NEXT: pmullw %xmm4, %xmm5883; SSE2-NEXT: pand %xmm8, %xmm5884; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]885; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]886; SSE2-NEXT: pmullw %xmm7, %xmm3887; SSE2-NEXT: pand %xmm8, %xmm3888; SSE2-NEXT: packuswb %xmm5, %xmm3889; SSE2-NEXT: retq890;891; SSE41-LABEL: mul_v64i8:892; SSE41: # %bb.0: # %entry893; SSE41-NEXT: movdqa %xmm0, %xmm9894; SSE41-NEXT: pmullw %xmm4, %xmm9895; SSE41-NEXT: pmovzxbw {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]896; SSE41-NEXT: pand %xmm8, %xmm9897; SSE41-NEXT: movdqa %xmm8, %xmm10898; SSE41-NEXT: pandn %xmm4, %xmm10899; SSE41-NEXT: pmaddubsw %xmm10, %xmm0900; SSE41-NEXT: psllw $8, %xmm0901; SSE41-NEXT: por %xmm9, %xmm0902; SSE41-NEXT: movdqa %xmm1, %xmm4903; SSE41-NEXT: pmullw %xmm5, %xmm4904; SSE41-NEXT: pand %xmm8, %xmm4905; SSE41-NEXT: movdqa %xmm8, %xmm9906; SSE41-NEXT: pandn %xmm5, %xmm9907; SSE41-NEXT: pmaddubsw %xmm9, %xmm1908; SSE41-NEXT: psllw $8, %xmm1909; SSE41-NEXT: por %xmm4, %xmm1910; SSE41-NEXT: movdqa %xmm2, %xmm4911; SSE41-NEXT: pmullw %xmm6, %xmm4912; SSE41-NEXT: pand %xmm8, %xmm4913; SSE41-NEXT: movdqa %xmm8, %xmm5914; SSE41-NEXT: pandn %xmm6, %xmm5915; SSE41-NEXT: pmaddubsw %xmm5, %xmm2916; SSE41-NEXT: psllw $8, %xmm2917; SSE41-NEXT: por %xmm4, %xmm2918; SSE41-NEXT: movdqa %xmm3, %xmm4919; SSE41-NEXT: pmullw %xmm7, %xmm4920; SSE41-NEXT: pand %xmm8, %xmm4921; SSE41-NEXT: pandn %xmm7, %xmm8922; SSE41-NEXT: pmaddubsw %xmm8, %xmm3923; SSE41-NEXT: psllw $8, %xmm3924; SSE41-NEXT: por %xmm4, %xmm3925; SSE41-NEXT: retq926;927; AVX2-LABEL: mul_v64i8:928; AVX2: # %bb.0: # %entry929; AVX2-NEXT: vpmullw %ymm2, %ymm0, %ymm4930; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]931; AVX2-NEXT: vpand %ymm5, %ymm4, %ymm4932; AVX2-NEXT: vpandn %ymm2, %ymm5, %ymm2933; AVX2-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm0934; AVX2-NEXT: vpsllw $8, %ymm0, %ymm0935; AVX2-NEXT: vpor %ymm0, %ymm4, %ymm0936; AVX2-NEXT: vpmullw %ymm3, %ymm1, %ymm2937; AVX2-NEXT: vpand %ymm5, %ymm2, %ymm2938; AVX2-NEXT: vpandn %ymm3, %ymm5, %ymm3939; AVX2-NEXT: vpmaddubsw %ymm3, %ymm1, %ymm1940; AVX2-NEXT: vpsllw $8, %ymm1, %ymm1941; AVX2-NEXT: vpor %ymm1, %ymm2, %ymm1942; AVX2-NEXT: retq943;944; AVX512F-LABEL: mul_v64i8:945; AVX512F: # %bb.0: # %entry946; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2947; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3948; AVX512F-NEXT: vpmullw %ymm2, %ymm3, %ymm4949; AVX512F-NEXT: vpmullw %ymm1, %ymm0, %ymm5950; AVX512F-NEXT: vinserti64x4 $1, %ymm4, %zmm5, %zmm4951; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm5 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]952; AVX512F-NEXT: vpandn %ymm1, %ymm5, %ymm1953; AVX512F-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0954; AVX512F-NEXT: vpsllw $8, %ymm0, %ymm0955; AVX512F-NEXT: vpandn %ymm2, %ymm5, %ymm1956; AVX512F-NEXT: vpmaddubsw %ymm1, %ymm3, %ymm1957; AVX512F-NEXT: vpsllw $8, %ymm1, %ymm1958; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0959; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm4 & zmm5)960; AVX512F-NEXT: retq961;962; AVX512BW-LABEL: mul_v64i8:963; AVX512BW: # %bb.0: # %entry964; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm2965; AVX512BW-NEXT: vpbroadcastw {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]966; AVX512BW-NEXT: vpandnq %zmm1, %zmm3, %zmm1967; AVX512BW-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0968; AVX512BW-NEXT: vpsllw $8, %zmm0, %zmm0969; AVX512BW-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & zmm3)970; AVX512BW-NEXT: retq971entry:972 %A = mul <64 x i8> %i, %j973 ret <64 x i8> %A974}975 976; PR30845977define <4 x i32> @mul_v4i64_zero_upper(<4 x i32> %val1, <4 x i32> %val2) {978; SSE2-LABEL: mul_v4i64_zero_upper:979; SSE2: # %bb.0: # %entry980; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,1,1,3]981; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,1,3,3]982; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,1,1,3]983; SSE2-NEXT: pmuludq %xmm2, %xmm0984; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,3,3]985; SSE2-NEXT: pmuludq %xmm3, %xmm1986; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]987; SSE2-NEXT: retq988;989; SSE41-LABEL: mul_v4i64_zero_upper:990; SSE41: # %bb.0: # %entry991; SSE41-NEXT: pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero992; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,1,3,3]993; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero994; SSE41-NEXT: pmuludq %xmm2, %xmm0995; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,3,3]996; SSE41-NEXT: pmuludq %xmm3, %xmm1997; SSE41-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]998; SSE41-NEXT: retq999;1000; AVX-LABEL: mul_v4i64_zero_upper:1001; AVX: # %bb.0: # %entry1002; AVX-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero1003; AVX-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero1004; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm01005; AVX-NEXT: vextracti128 $1, %ymm0, %xmm11006; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]1007; AVX-NEXT: vzeroupper1008; AVX-NEXT: retq1009entry:1010 %val1a = zext <4 x i32> %val1 to <4 x i64>1011 %val2a = zext <4 x i32> %val2 to <4 x i64>1012 %res64 = mul <4 x i64> %val1a, %val2a1013 %rescast = bitcast <4 x i64> %res64 to <8 x i32>1014 %res = shufflevector <8 x i32> %rescast, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>1015 ret <4 x i32> %res1016}1017 1018define <4 x i32> @mul_v4i64_zero_upper_left(<4 x i32> %val1, <4 x i64> %val2) {1019; SSE2-LABEL: mul_v4i64_zero_upper_left:1020; SSE2: # %bb.0: # %entry1021; SSE2-NEXT: pxor %xmm4, %xmm41022; SSE2-NEXT: movdqa %xmm0, %xmm31023; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]1024; SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]1025; SSE2-NEXT: movdqa %xmm0, %xmm41026; SSE2-NEXT: pmuludq %xmm2, %xmm41027; SSE2-NEXT: psrlq $32, %xmm21028; SSE2-NEXT: pmuludq %xmm0, %xmm21029; SSE2-NEXT: psllq $32, %xmm21030; SSE2-NEXT: paddq %xmm4, %xmm21031; SSE2-NEXT: movdqa %xmm3, %xmm01032; SSE2-NEXT: pmuludq %xmm1, %xmm01033; SSE2-NEXT: psrlq $32, %xmm11034; SSE2-NEXT: pmuludq %xmm1, %xmm31035; SSE2-NEXT: psllq $32, %xmm31036; SSE2-NEXT: paddq %xmm0, %xmm31037; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm2[1,3]1038; SSE2-NEXT: movaps %xmm3, %xmm01039; SSE2-NEXT: retq1040;1041; SSE41-LABEL: mul_v4i64_zero_upper_left:1042; SSE41: # %bb.0: # %entry1043; SSE41-NEXT: pxor %xmm4, %xmm41044; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero1045; SSE41-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]1046; SSE41-NEXT: movdqa %xmm0, %xmm41047; SSE41-NEXT: pmuludq %xmm2, %xmm41048; SSE41-NEXT: psrlq $32, %xmm21049; SSE41-NEXT: pmuludq %xmm0, %xmm21050; SSE41-NEXT: psllq $32, %xmm21051; SSE41-NEXT: paddq %xmm4, %xmm21052; SSE41-NEXT: movdqa %xmm3, %xmm01053; SSE41-NEXT: pmuludq %xmm1, %xmm01054; SSE41-NEXT: psrlq $32, %xmm11055; SSE41-NEXT: pmuludq %xmm1, %xmm31056; SSE41-NEXT: psllq $32, %xmm31057; SSE41-NEXT: paddq %xmm0, %xmm31058; SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,3],xmm2[1,3]1059; SSE41-NEXT: movaps %xmm3, %xmm01060; SSE41-NEXT: retq1061;1062; AVX-LABEL: mul_v4i64_zero_upper_left:1063; AVX: # %bb.0: # %entry1064; AVX-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero1065; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm21066; AVX-NEXT: vpsrlq $32, %ymm1, %ymm11067; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm01068; AVX-NEXT: vpsllq $32, %ymm0, %ymm01069; AVX-NEXT: vpaddq %ymm0, %ymm2, %ymm01070; AVX-NEXT: vextracti128 $1, %ymm0, %xmm11071; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]1072; AVX-NEXT: vzeroupper1073; AVX-NEXT: retq1074entry:1075 %val1a = zext <4 x i32> %val1 to <4 x i64>1076 %res64 = mul <4 x i64> %val1a, %val21077 %rescast = bitcast <4 x i64> %res64 to <8 x i32>1078 %res = shufflevector <8 x i32> %rescast, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>1079 ret <4 x i32> %res1080}1081 1082define <4 x i32> @mul_v4i64_zero_lower(<4 x i32> %val1, <4 x i64> %val2) {1083; SSE2-LABEL: mul_v4i64_zero_lower:1084; SSE2: # %bb.0: # %entry1085; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,1,1,3]1086; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,3,3]1087; SSE2-NEXT: psrlq $32, %xmm21088; SSE2-NEXT: pmuludq %xmm0, %xmm21089; SSE2-NEXT: psrlq $32, %xmm11090; SSE2-NEXT: pmuludq %xmm1, %xmm31091; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,2],xmm2[0,2]1092; SSE2-NEXT: movaps %xmm3, %xmm01093; SSE2-NEXT: retq1094;1095; SSE41-LABEL: mul_v4i64_zero_lower:1096; SSE41: # %bb.0: # %entry1097; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero1098; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,3,3]1099; SSE41-NEXT: psrlq $32, %xmm21100; SSE41-NEXT: pmuludq %xmm0, %xmm21101; SSE41-NEXT: psrlq $32, %xmm11102; SSE41-NEXT: pmuludq %xmm1, %xmm31103; SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,2],xmm2[0,2]1104; SSE41-NEXT: movaps %xmm3, %xmm01105; SSE41-NEXT: retq1106;1107; AVX-LABEL: mul_v4i64_zero_lower:1108; AVX: # %bb.0: # %entry1109; AVX-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero1110; AVX-NEXT: vpsrlq $32, %ymm1, %ymm11111; AVX-NEXT: vpmuludq %ymm1, %ymm0, %ymm01112; AVX-NEXT: vextracti128 $1, %ymm0, %xmm11113; AVX-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]1114; AVX-NEXT: vzeroupper1115; AVX-NEXT: retq1116entry:1117 %val1a = zext <4 x i32> %val1 to <4 x i64>1118 %val2a = and <4 x i64> %val2, <i64 -4294967296, i64 -4294967296, i64 -4294967296, i64 -4294967296>1119 %res64 = mul <4 x i64> %val1a, %val2a1120 %rescast = bitcast <4 x i64> %res64 to <8 x i32>1121 %res = shufflevector <8 x i32> %rescast, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>1122 ret <4 x i32> %res1123}1124 1125define <8 x i32> @mul_v8i64_zero_upper(<8 x i32> %val1, <8 x i32> %val2) {1126; SSE2-LABEL: mul_v8i64_zero_upper:1127; SSE2: # %bb.0: # %entry1128; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,1,1,3]1129; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm0[2,1,3,3]1130; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm1[0,1,1,3]1131; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm1[2,1,3,3]1132; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,1,1,3]1133; SSE2-NEXT: pmuludq %xmm4, %xmm01134; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3]1135; SSE2-NEXT: pmuludq %xmm5, %xmm11136; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]1137; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,1,1,3]1138; SSE2-NEXT: pmuludq %xmm6, %xmm11139; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,1,3,3]1140; SSE2-NEXT: pmuludq %xmm7, %xmm21141; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,3],xmm2[1,3]1142; SSE2-NEXT: retq1143;1144; SSE41-LABEL: mul_v8i64_zero_upper:1145; SSE41: # %bb.0: # %entry1146; SSE41-NEXT: pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero1147; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[2,1,3,3]1148; SSE41-NEXT: pmovzxdq {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero1149; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm1[2,1,3,3]1150; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero1151; SSE41-NEXT: pmuludq %xmm4, %xmm01152; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3]1153; SSE41-NEXT: pmuludq %xmm5, %xmm11154; SSE41-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]1155; SSE41-NEXT: pmovzxdq {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero1156; SSE41-NEXT: pmuludq %xmm6, %xmm11157; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,1,3,3]1158; SSE41-NEXT: pmuludq %xmm7, %xmm21159; SSE41-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,3],xmm2[1,3]1160; SSE41-NEXT: retq1161;1162; AVX2-LABEL: mul_v8i64_zero_upper:1163; AVX2: # %bb.0: # %entry1164; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero1165; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm01166; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero1167; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero1168; AVX2-NEXT: vpmuludq %ymm3, %ymm2, %ymm21169; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm11170; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero1171; AVX2-NEXT: vpmuludq %ymm1, %ymm0, %ymm01172; AVX2-NEXT: vshufps {{.*#+}} ymm0 = ymm2[1,3],ymm0[1,3],ymm2[5,7],ymm0[5,7]1173; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]1174; AVX2-NEXT: retq1175;1176; AVX512-LABEL: mul_v8i64_zero_upper:1177; AVX512: # %bb.0: # %entry1178; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero1179; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero1180; AVX512-NEXT: vpmuludq %zmm1, %zmm0, %zmm01181; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm11182; AVX512-NEXT: vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]1183; AVX512-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]1184; AVX512-NEXT: retq1185entry:1186 %val1a = zext <8 x i32> %val1 to <8 x i64>1187 %val2a = zext <8 x i32> %val2 to <8 x i64>1188 %res64 = mul <8 x i64> %val1a, %val2a1189 %rescast = bitcast <8 x i64> %res64 to <16 x i32>1190 %res = shufflevector <16 x i32> %rescast, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7,i32 9, i32 11, i32 13, i32 15 >1191 ret <8 x i32> %res1192}1193 1194define <8 x i64> @mul_v8i64_sext(<8 x i16> %val1, <8 x i32> %val2) {1195; SSE2-LABEL: mul_v8i64_sext:1196; SSE2: # %bb.0:1197; SSE2-NEXT: movdqa %xmm1, %xmm41198; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]1199; SSE2-NEXT: psrad $16, %xmm61200; SSE2-NEXT: pxor %xmm12, %xmm121201; SSE2-NEXT: pxor %xmm7, %xmm71202; SSE2-NEXT: pcmpgtd %xmm6, %xmm71203; SSE2-NEXT: movdqa %xmm6, %xmm51204; SSE2-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm7[2],xmm5[3],xmm7[3]1205; SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]1206; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]1207; SSE2-NEXT: psrad $16, %xmm01208; SSE2-NEXT: pxor %xmm11, %xmm111209; SSE2-NEXT: pcmpgtd %xmm0, %xmm111210; SSE2-NEXT: movdqa %xmm0, %xmm91211; SSE2-NEXT: punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm11[2],xmm9[3],xmm11[3]1212; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm11[0],xmm0[1],xmm11[1]1213; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]1214; SSE2-NEXT: pxor %xmm8, %xmm81215; SSE2-NEXT: pcmpgtd %xmm3, %xmm81216; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1]1217; SSE2-NEXT: pxor %xmm10, %xmm101218; SSE2-NEXT: pcmpgtd %xmm2, %xmm101219; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1]1220; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]1221; SSE2-NEXT: pxor %xmm13, %xmm131222; SSE2-NEXT: pcmpgtd %xmm1, %xmm131223; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm13[0],xmm1[1],xmm13[1]1224; SSE2-NEXT: pcmpgtd %xmm4, %xmm121225; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm12[0],xmm4[1],xmm12[1]1226; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm11[0,1,1,3]1227; SSE2-NEXT: pmuludq %xmm4, %xmm141228; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm12[0,1,1,3]1229; SSE2-NEXT: pmuludq %xmm0, %xmm121230; SSE2-NEXT: paddq %xmm14, %xmm121231; SSE2-NEXT: psllq $32, %xmm121232; SSE2-NEXT: pmuludq %xmm4, %xmm01233; SSE2-NEXT: paddq %xmm12, %xmm01234; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm11[2,1,3,3]1235; SSE2-NEXT: pmuludq %xmm1, %xmm41236; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm13[0,1,1,3]1237; SSE2-NEXT: pmuludq %xmm9, %xmm111238; SSE2-NEXT: paddq %xmm4, %xmm111239; SSE2-NEXT: psllq $32, %xmm111240; SSE2-NEXT: pmuludq %xmm9, %xmm11241; SSE2-NEXT: paddq %xmm11, %xmm11242; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,1,1,3]1243; SSE2-NEXT: pmuludq %xmm2, %xmm41244; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm10[0,1,1,3]1245; SSE2-NEXT: pmuludq %xmm6, %xmm91246; SSE2-NEXT: paddq %xmm4, %xmm91247; SSE2-NEXT: psllq $32, %xmm91248; SSE2-NEXT: pmuludq %xmm6, %xmm21249; SSE2-NEXT: paddq %xmm9, %xmm21250; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm7[2,1,3,3]1251; SSE2-NEXT: pmuludq %xmm3, %xmm41252; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm8[0,1,1,3]1253; SSE2-NEXT: pmuludq %xmm5, %xmm61254; SSE2-NEXT: paddq %xmm4, %xmm61255; SSE2-NEXT: psllq $32, %xmm61256; SSE2-NEXT: pmuludq %xmm5, %xmm31257; SSE2-NEXT: paddq %xmm6, %xmm31258; SSE2-NEXT: retq1259;1260; SSE41-LABEL: mul_v8i64_sext:1261; SSE41: # %bb.0:1262; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]1263; SSE41-NEXT: pmovsxwq %xmm3, %xmm41264; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]1265; SSE41-NEXT: pmovsxwq %xmm3, %xmm51266; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]1267; SSE41-NEXT: pmovsxwq %xmm3, %xmm61268; SSE41-NEXT: pmovsxwq %xmm0, %xmm71269; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,1,3,3]1270; SSE41-NEXT: pmuldq %xmm4, %xmm31271; SSE41-NEXT: pmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero1272; SSE41-NEXT: pmuldq %xmm5, %xmm21273; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,1,3,3]1274; SSE41-NEXT: pmuldq %xmm6, %xmm41275; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero1276; SSE41-NEXT: pmuldq %xmm7, %xmm01277; SSE41-NEXT: movdqa %xmm4, %xmm11278; SSE41-NEXT: retq1279;1280; AVX2-LABEL: mul_v8i64_sext:1281; AVX2: # %bb.0:1282; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]1283; AVX2-NEXT: vpmovsxwq %xmm2, %ymm21284; AVX2-NEXT: vpmovsxwq %xmm0, %ymm01285; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm31286; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero1287; AVX2-NEXT: vpmuldq %ymm3, %ymm2, %ymm21288; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero1289; AVX2-NEXT: vpmuldq %ymm1, %ymm0, %ymm01290; AVX2-NEXT: vmovdqa %ymm2, %ymm11291; AVX2-NEXT: retq1292;1293; AVX512-LABEL: mul_v8i64_sext:1294; AVX512: # %bb.0:1295; AVX512-NEXT: vpmovsxwq %xmm0, %zmm01296; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero1297; AVX512-NEXT: vpmuldq %zmm1, %zmm0, %zmm01298; AVX512-NEXT: retq1299 %1 = sext <8 x i16> %val1 to <8 x i64>1300 %2 = sext <8 x i32> %val2 to <8 x i64>1301 %3 = mul <8 x i64> %1, %21302 ret <8 x i64> %31303}1304 1305define <2 x i64> @pmuldq_square(<2 x i64> %x) {1306; SSE2-LABEL: pmuldq_square:1307; SSE2: # %bb.0:1308; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]1309; SSE2-NEXT: psllq $32, %xmm01310; SSE2-NEXT: psrad $31, %xmm01311; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]1312; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]1313; SSE2-NEXT: psrlq $32, %xmm01314; SSE2-NEXT: pmuludq %xmm1, %xmm01315; SSE2-NEXT: pmuludq %xmm1, %xmm11316; SSE2-NEXT: psllq $33, %xmm01317; SSE2-NEXT: paddq %xmm1, %xmm01318; SSE2-NEXT: retq1319;1320; SSE41-LABEL: pmuldq_square:1321; SSE41: # %bb.0:1322; SSE41-NEXT: pmuldq %xmm0, %xmm01323; SSE41-NEXT: retq1324;1325; AVX-LABEL: pmuldq_square:1326; AVX: # %bb.0:1327; AVX-NEXT: vpmuldq %xmm0, %xmm0, %xmm01328; AVX-NEXT: retq1329 %1 = shl <2 x i64> %x, <i64 32, i64 32>1330 %2 = ashr exact <2 x i64> %1, <i64 32, i64 32>1331 %3 = mul nsw <2 x i64> %2, %21332 ret <2 x i64> %31333}1334 1335define <2 x i64> @pmuludq_square(<2 x i64> %x) {1336; SSE-LABEL: pmuludq_square:1337; SSE: # %bb.0:1338; SSE-NEXT: pmuludq %xmm0, %xmm01339; SSE-NEXT: retq1340;1341; AVX-LABEL: pmuludq_square:1342; AVX: # %bb.0:1343; AVX-NEXT: vpmuludq %xmm0, %xmm0, %xmm01344; AVX-NEXT: retq1345 %1 = and <2 x i64> %x, <i64 4294967295, i64 4294967295>1346 %2 = mul nuw <2 x i64> %1, %11347 ret <2 x i64> %21348}1349