brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=SSE,SSE23; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE414; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX25; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BW7 8define <4 x i16> @zext_mulhuw_v4i16(<4 x i16> %a, <4 x i16> %b) {9; SSE-LABEL: zext_mulhuw_v4i16:10; SSE:       # %bb.0:11; SSE-NEXT:    pmulhuw %xmm1, %xmm012; SSE-NEXT:    retq13;14; AVX-LABEL: zext_mulhuw_v4i16:15; AVX:       # %bb.0:16; AVX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm017; AVX-NEXT:    retq18  %a1 = zext <4 x i16> %a to <4 x i32>19  %b1 = zext <4 x i16> %b to <4 x i32>20  %c = mul <4 x i32> %a1, %b121  %d = lshr <4 x i32> %c, <i32 16, i32 16, i32 16, i32 16>22  %e = trunc <4 x i32> %d to <4 x i16>23  ret <4 x i16> %e24}25 26define <4 x i16> @and_mulhuw_v4i16(<4 x i64> %a, <4 x i64> %b) {27; SSE2-LABEL: and_mulhuw_v4i16:28; SSE2:       # %bb.0:29; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]30; SSE2-NEXT:    pslld $16, %xmm231; SSE2-NEXT:    psrad $16, %xmm232; SSE2-NEXT:    xorps %xmm3, %xmm333; SSE2-NEXT:    packssdw %xmm3, %xmm234; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]35; SSE2-NEXT:    pslld $16, %xmm036; SSE2-NEXT:    psrad $16, %xmm037; SSE2-NEXT:    packssdw %xmm3, %xmm038; SSE2-NEXT:    pmulhuw %xmm2, %xmm039; SSE2-NEXT:    retq40;41; SSE41-LABEL: and_mulhuw_v4i16:42; SSE41:       # %bb.0:43; SSE41-NEXT:    pxor %xmm4, %xmm444; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1,2,3],xmm3[4],xmm4[5,6,7]45; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1,2,3],xmm2[4],xmm4[5,6,7]46; SSE41-NEXT:    packusdw %xmm3, %xmm247; SSE41-NEXT:    packusdw %xmm4, %xmm248; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm4[1,2,3],xmm1[4],xmm4[5,6,7]49; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm4[1,2,3],xmm0[4],xmm4[5,6,7]50; SSE41-NEXT:    packusdw %xmm1, %xmm051; SSE41-NEXT:    packusdw %xmm4, %xmm052; SSE41-NEXT:    pmulhuw %xmm2, %xmm053; SSE41-NEXT:    retq54;55; AVX2-LABEL: and_mulhuw_v4i16:56; AVX2:       # %bb.0:57; AVX2-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm058; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm159; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15]60; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm161; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm062; AVX2-NEXT:    vpackusdw %xmm0, %xmm0, %xmm063; AVX2-NEXT:    vzeroupper64; AVX2-NEXT:    retq65;66; AVX512-LABEL: and_mulhuw_v4i16:67; AVX512:       # %bb.0:68; AVX512-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm069; AVX512-NEXT:    vpmovqw %zmm0, %xmm070; AVX512-NEXT:    vzeroupper71; AVX512-NEXT:    retq72  %a1 = and <4 x i64> %a, <i64 65535, i64 65535, i64 65535, i64 65535>73  %b1 = and <4 x i64> %b, <i64 65535, i64 65535, i64 65535, i64 65535>74  %c = mul <4 x i64> %a1, %b175  %d = lshr <4 x i64> %c, <i64 16, i64 16, i64 16, i64 16>76  %e = trunc <4 x i64> %d to <4 x i16>77  ret <4 x i16> %e78}79 80define <4 x i16> @sext_mulhw_v4i16(<4 x i16> %a, <4 x i16> %b) {81; SSE-LABEL: sext_mulhw_v4i16:82; SSE:       # %bb.0:83; SSE-NEXT:    pmulhw %xmm1, %xmm084; SSE-NEXT:    retq85;86; AVX-LABEL: sext_mulhw_v4i16:87; AVX:       # %bb.0:88; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm089; AVX-NEXT:    retq90  %a1 = sext <4 x i16> %a to <4 x i32>91  %b1 = sext <4 x i16> %b to <4 x i32>92  %c = mul <4 x i32> %a1, %b193  %d = lshr <4 x i32> %c, <i32 16, i32 16, i32 16, i32 16>94  %e = trunc <4 x i32> %d to <4 x i16>95  ret <4 x i16> %e96}97 98define <4 x i16> @ashr_mulhw_v4i16(<4 x i32> %a, <4 x i32> %b) {99; SSE2-LABEL: ashr_mulhw_v4i16:100; SSE2:       # %bb.0:101; SSE2-NEXT:    psrad $16, %xmm1102; SSE2-NEXT:    packssdw %xmm1, %xmm1103; SSE2-NEXT:    psrad $16, %xmm0104; SSE2-NEXT:    packssdw %xmm0, %xmm0105; SSE2-NEXT:    pmulhw %xmm1, %xmm0106; SSE2-NEXT:    retq107;108; SSE41-LABEL: ashr_mulhw_v4i16:109; SSE41:       # %bb.0:110; SSE41-NEXT:    psrld $16, %xmm0111; SSE41-NEXT:    psrld $16, %xmm1112; SSE41-NEXT:    packusdw %xmm1, %xmm1113; SSE41-NEXT:    packusdw %xmm0, %xmm0114; SSE41-NEXT:    pmulhw %xmm1, %xmm0115; SSE41-NEXT:    retq116;117; AVX-LABEL: ashr_mulhw_v4i16:118; AVX:       # %bb.0:119; AVX-NEXT:    vpsrld $16, %xmm0, %xmm0120; AVX-NEXT:    vpsrld $16, %xmm1, %xmm1121; AVX-NEXT:    vpackusdw %xmm1, %xmm1, %xmm1122; AVX-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0123; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0124; AVX-NEXT:    retq125  %a1 = ashr <4 x i32> %a, <i32 16, i32 16, i32 16, i32 16>126  %b1 = ashr <4 x i32> %b, <i32 16, i32 16, i32 16, i32 16>127  %c = mul <4 x i32> %a1, %b1128  %d = lshr <4 x i32> %c, <i32 16, i32 16, i32 16, i32 16>129  %e = trunc <4 x i32> %d to <4 x i16>130  ret <4 x i16> %e131}132 133define <8 x i16> @zext_mulhuw_v8i16(<8 x i16> %a, <8 x i16> %b) {134; SSE-LABEL: zext_mulhuw_v8i16:135; SSE:       # %bb.0:136; SSE-NEXT:    pmulhuw %xmm1, %xmm0137; SSE-NEXT:    retq138;139; AVX-LABEL: zext_mulhuw_v8i16:140; AVX:       # %bb.0:141; AVX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0142; AVX-NEXT:    retq143  %a1 = zext <8 x i16> %a to <8 x i32>144  %b1 = zext <8 x i16> %b to <8 x i32>145  %c = mul <8 x i32> %a1, %b1146  %d = lshr <8 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>147  %e = trunc <8 x i32> %d to <8 x i16>148  ret <8 x i16> %e149}150 151define <8 x i16> @lshr_mulhuw_v8i16(<8 x i32> %a, <8 x i32> %b) {152; SSE2-LABEL: lshr_mulhuw_v8i16:153; SSE2:       # %bb.0:154; SSE2-NEXT:    psrad $16, %xmm3155; SSE2-NEXT:    psrad $16, %xmm2156; SSE2-NEXT:    packssdw %xmm3, %xmm2157; SSE2-NEXT:    psrad $16, %xmm1158; SSE2-NEXT:    psrad $16, %xmm0159; SSE2-NEXT:    packssdw %xmm1, %xmm0160; SSE2-NEXT:    pmulhuw %xmm2, %xmm0161; SSE2-NEXT:    retq162;163; SSE41-LABEL: lshr_mulhuw_v8i16:164; SSE41:       # %bb.0:165; SSE41-NEXT:    psrld $16, %xmm1166; SSE41-NEXT:    psrld $16, %xmm0167; SSE41-NEXT:    packusdw %xmm1, %xmm0168; SSE41-NEXT:    psrld $16, %xmm3169; SSE41-NEXT:    psrld $16, %xmm2170; SSE41-NEXT:    packusdw %xmm3, %xmm2171; SSE41-NEXT:    pmulhuw %xmm2, %xmm0172; SSE41-NEXT:    retq173;174; AVX2-LABEL: lshr_mulhuw_v8i16:175; AVX2:       # %bb.0:176; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0177; AVX2-NEXT:    vpsrld $16, %ymm1, %ymm1178; AVX2-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0179; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1180; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0181; AVX2-NEXT:    vzeroupper182; AVX2-NEXT:    retq183;184; AVX512-LABEL: lshr_mulhuw_v8i16:185; AVX512:       # %bb.0:186; AVX512-NEXT:    vpsrld $16, %ymm0, %ymm0187; AVX512-NEXT:    vpsrld $16, %ymm1, %ymm1188; AVX512-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0189; AVX512-NEXT:    vpmovdw %zmm0, %ymm0190; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0191; AVX512-NEXT:    vzeroupper192; AVX512-NEXT:    retq193  %a1 = lshr <8 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>194  %b1 = lshr <8 x i32> %b, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>195  %c = mul <8 x i32> %a1, %b1196  %d = lshr <8 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>197  %e = trunc <8 x i32> %d to <8 x i16>198  ret <8 x i16> %e199}200 201define <8 x i16> @sext_mulhw_v8i16(<8 x i16> %a, <8 x i16> %b) {202; SSE-LABEL: sext_mulhw_v8i16:203; SSE:       # %bb.0:204; SSE-NEXT:    pmulhw %xmm1, %xmm0205; SSE-NEXT:    retq206;207; AVX-LABEL: sext_mulhw_v8i16:208; AVX:       # %bb.0:209; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0210; AVX-NEXT:    retq211  %a1 = sext <8 x i16> %a to <8 x i32>212  %b1 = sext <8 x i16> %b to <8 x i32>213  %c = mul <8 x i32> %a1, %b1214  %d = lshr <8 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>215  %e = trunc <8 x i32> %d to <8 x i16>216  ret <8 x i16> %e217}218 219define <8 x i16> @sextinreg_mulhw_v8i16(<8 x i32> %a, <8 x i32> %b) {220; SSE-LABEL: sextinreg_mulhw_v8i16:221; SSE:       # %bb.0:222; SSE-NEXT:    pslld $24, %xmm1223; SSE-NEXT:    psrad $24, %xmm1224; SSE-NEXT:    pslld $24, %xmm0225; SSE-NEXT:    psrad $24, %xmm0226; SSE-NEXT:    packssdw %xmm1, %xmm0227; SSE-NEXT:    pslld $25, %xmm3228; SSE-NEXT:    psrad $25, %xmm3229; SSE-NEXT:    pslld $25, %xmm2230; SSE-NEXT:    psrad $25, %xmm2231; SSE-NEXT:    packssdw %xmm3, %xmm2232; SSE-NEXT:    pmulhw %xmm2, %xmm0233; SSE-NEXT:    retq234;235; AVX2-LABEL: sextinreg_mulhw_v8i16:236; AVX2:       # %bb.0:237; AVX2-NEXT:    vpslld $24, %ymm0, %ymm0238; AVX2-NEXT:    vpsrad $24, %ymm0, %ymm0239; AVX2-NEXT:    vpslld $25, %ymm1, %ymm1240; AVX2-NEXT:    vpsrad $25, %ymm1, %ymm1241; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2242; AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1243; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2244; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0245; AVX2-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0246; AVX2-NEXT:    vzeroupper247; AVX2-NEXT:    retq248;249; AVX512-LABEL: sextinreg_mulhw_v8i16:250; AVX512:       # %bb.0:251; AVX512-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1252; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0253; AVX512-NEXT:    vpmovdw %zmm1, %ymm1254; AVX512-NEXT:    vpsllw $9, %xmm1, %xmm1255; AVX512-NEXT:    vpsraw $9, %xmm1, %xmm1256; AVX512-NEXT:    vpmovdw %zmm0, %ymm0257; AVX512-NEXT:    vpsllw $8, %xmm0, %xmm0258; AVX512-NEXT:    vpsraw $8, %xmm0, %xmm0259; AVX512-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0260; AVX512-NEXT:    vzeroupper261; AVX512-NEXT:    retq262  %a1 = shl <8 x i32> %a, <i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24>263  %b1 = shl <8 x i32> %b, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>264  %a2 = ashr <8 x i32> %a1, <i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24>265  %b2 = ashr <8 x i32> %b1, <i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25, i32 25>266  %c = mul <8 x i32> %a2, %b2267  %d = lshr <8 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>268  %e = trunc <8 x i32> %d to <8 x i16>269  ret <8 x i16> %e270}271 272define <8 x i16> @zext_mulhuw_v8i16_v8i33(<8 x i16> %a, <8 x i16> %b) {273; SSE-LABEL: zext_mulhuw_v8i16_v8i33:274; SSE:       # %bb.0:275; SSE-NEXT:    pmulhuw %xmm1, %xmm0276; SSE-NEXT:    retq277;278; AVX-LABEL: zext_mulhuw_v8i16_v8i33:279; AVX:       # %bb.0:280; AVX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0281; AVX-NEXT:    retq282  %a1 = zext <8 x i16> %a to <8 x i33>283  %b1 = zext <8 x i16> %b to <8 x i33>284  %c = mul <8 x i33> %a1, %b1285  %d = lshr <8 x i33> %c, <i33 16, i33 16, i33 16, i33 16, i33 16, i33 16, i33 16, i33 16>286  %e = trunc <8 x i33> %d to <8 x i16>287  ret <8 x i16> %e288}289 290define <16 x i16> @zext_mulhuw_v16i16(<16 x i16> %a, <16 x i16> %b) {291; SSE-LABEL: zext_mulhuw_v16i16:292; SSE:       # %bb.0:293; SSE-NEXT:    pmulhuw %xmm2, %xmm0294; SSE-NEXT:    pmulhuw %xmm3, %xmm1295; SSE-NEXT:    retq296;297; AVX-LABEL: zext_mulhuw_v16i16:298; AVX:       # %bb.0:299; AVX-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0300; AVX-NEXT:    retq301  %a1 = zext <16 x i16> %a to <16 x i32>302  %b1 = zext <16 x i16> %b to <16 x i32>303  %c = mul <16 x i32> %a1, %b1304  %d = lshr <16 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>305  %e = trunc <16 x i32> %d to <16 x i16>306  ret <16 x i16> %e307}308 309define <16 x i16> @and_mulhuw_v16i16(<16 x i32> %a, <16 x i32> %b) {310; SSE2-LABEL: and_mulhuw_v16i16:311; SSE2:       # %bb.0:312; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [32767,32767,32767,32767]313; SSE2-NEXT:    pand %xmm8, %xmm3314; SSE2-NEXT:    pand %xmm8, %xmm2315; SSE2-NEXT:    packssdw %xmm3, %xmm2316; SSE2-NEXT:    pand %xmm8, %xmm1317; SSE2-NEXT:    pand %xmm8, %xmm0318; SSE2-NEXT:    packssdw %xmm1, %xmm0319; SSE2-NEXT:    pand %xmm8, %xmm7320; SSE2-NEXT:    pand %xmm8, %xmm6321; SSE2-NEXT:    packssdw %xmm7, %xmm6322; SSE2-NEXT:    pmulhw %xmm2, %xmm6323; SSE2-NEXT:    pand %xmm8, %xmm5324; SSE2-NEXT:    pand %xmm4, %xmm8325; SSE2-NEXT:    packssdw %xmm5, %xmm8326; SSE2-NEXT:    pmulhw %xmm8, %xmm0327; SSE2-NEXT:    movdqa %xmm6, %xmm1328; SSE2-NEXT:    retq329;330; SSE41-LABEL: and_mulhuw_v16i16:331; SSE41:       # %bb.0:332; SSE41-NEXT:    pmovsxwd {{.*#+}} xmm8 = [32767,32767,32767,32767]333; SSE41-NEXT:    pand %xmm8, %xmm3334; SSE41-NEXT:    pand %xmm8, %xmm2335; SSE41-NEXT:    packusdw %xmm3, %xmm2336; SSE41-NEXT:    pand %xmm8, %xmm1337; SSE41-NEXT:    pand %xmm8, %xmm0338; SSE41-NEXT:    packusdw %xmm1, %xmm0339; SSE41-NEXT:    pand %xmm8, %xmm7340; SSE41-NEXT:    pand %xmm8, %xmm6341; SSE41-NEXT:    packusdw %xmm7, %xmm6342; SSE41-NEXT:    pmulhw %xmm2, %xmm6343; SSE41-NEXT:    pand %xmm8, %xmm5344; SSE41-NEXT:    pand %xmm4, %xmm8345; SSE41-NEXT:    packusdw %xmm5, %xmm8346; SSE41-NEXT:    pmulhw %xmm8, %xmm0347; SSE41-NEXT:    movdqa %xmm6, %xmm1348; SSE41-NEXT:    retq349;350; AVX2-LABEL: and_mulhuw_v16i16:351; AVX2:       # %bb.0:352; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm4 = [32767,32767,32767,32767,32767,32767,32767,32767]353; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0354; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1355; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2356; AVX2-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm0357; AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm2358; AVX2-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm1359; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0360; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]361; AVX2-NEXT:    retq362;363; AVX512F-LABEL: and_mulhuw_v16i16:364; AVX512F:       # %bb.0:365; AVX512F-NEXT:    vpmovdw %zmm1, %ymm1366; AVX512F-NEXT:    vpbroadcastw {{.*#+}} ymm2 = [32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767]367; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm1368; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0369; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm0370; AVX512F-NEXT:    vpmulhw %ymm1, %ymm0, %ymm0371; AVX512F-NEXT:    retq372;373; AVX512BW-LABEL: and_mulhuw_v16i16:374; AVX512BW:       # %bb.0:375; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} zmm2 = [32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767]376; AVX512BW-NEXT:    vpandd %zmm2, %zmm0, %zmm0377; AVX512BW-NEXT:    vpandd %zmm2, %zmm1, %zmm1378; AVX512BW-NEXT:    vpmulhuw %zmm1, %zmm0, %zmm0379; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0380; AVX512BW-NEXT:    retq381  %a1 = and <16 x i32> %a, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>382  %b1 = and <16 x i32> %b, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>383  %c = mul <16 x i32> %a1, %b1384  %d = lshr <16 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>385  %e = trunc <16 x i32> %d to <16 x i16>386  ret <16 x i16> %e387}388 389define <16 x i16> @sext_mulhuw_v16i16(<16 x i16> %a, <16 x i16> %b) {390; SSE-LABEL: sext_mulhuw_v16i16:391; SSE:       # %bb.0:392; SSE-NEXT:    pmulhw %xmm2, %xmm0393; SSE-NEXT:    pmulhw %xmm3, %xmm1394; SSE-NEXT:    retq395;396; AVX-LABEL: sext_mulhuw_v16i16:397; AVX:       # %bb.0:398; AVX-NEXT:    vpmulhw %ymm1, %ymm0, %ymm0399; AVX-NEXT:    retq400  %a1 = sext <16 x i16> %a to <16 x i32>401  %b1 = sext <16 x i16> %b to <16 x i32>402  %c = mul <16 x i32> %a1, %b1403  %d = lshr <16 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>404  %e = trunc <16 x i32> %d to <16 x i16>405  ret <16 x i16> %e406}407 408define <16 x i16> @ashr_mulhuw_v16i16(<16 x i32> %a, <16 x i32> %b) {409; SSE2-LABEL: ashr_mulhuw_v16i16:410; SSE2:       # %bb.0:411; SSE2-NEXT:    psrad $16, %xmm5412; SSE2-NEXT:    psrad $16, %xmm4413; SSE2-NEXT:    packssdw %xmm5, %xmm4414; SSE2-NEXT:    psrad $16, %xmm1415; SSE2-NEXT:    psrad $16, %xmm0416; SSE2-NEXT:    packssdw %xmm1, %xmm0417; SSE2-NEXT:    pmulhw %xmm4, %xmm0418; SSE2-NEXT:    psrad $16, %xmm7419; SSE2-NEXT:    psrad $16, %xmm6420; SSE2-NEXT:    packssdw %xmm7, %xmm6421; SSE2-NEXT:    psrad $16, %xmm3422; SSE2-NEXT:    psrad $16, %xmm2423; SSE2-NEXT:    packssdw %xmm3, %xmm2424; SSE2-NEXT:    pmulhw %xmm6, %xmm2425; SSE2-NEXT:    movdqa %xmm2, %xmm1426; SSE2-NEXT:    retq427;428; SSE41-LABEL: ashr_mulhuw_v16i16:429; SSE41:       # %bb.0:430; SSE41-NEXT:    psrld $16, %xmm3431; SSE41-NEXT:    psrld $16, %xmm2432; SSE41-NEXT:    packusdw %xmm3, %xmm2433; SSE41-NEXT:    psrld $16, %xmm1434; SSE41-NEXT:    psrld $16, %xmm0435; SSE41-NEXT:    packusdw %xmm1, %xmm0436; SSE41-NEXT:    psrld $16, %xmm7437; SSE41-NEXT:    psrld $16, %xmm6438; SSE41-NEXT:    packusdw %xmm7, %xmm6439; SSE41-NEXT:    pmulhw %xmm2, %xmm6440; SSE41-NEXT:    psrld $16, %xmm5441; SSE41-NEXT:    psrld $16, %xmm4442; SSE41-NEXT:    packusdw %xmm5, %xmm4443; SSE41-NEXT:    pmulhw %xmm4, %xmm0444; SSE41-NEXT:    movdqa %xmm6, %xmm1445; SSE41-NEXT:    retq446;447; AVX2-LABEL: ashr_mulhuw_v16i16:448; AVX2:       # %bb.0:449; AVX2-NEXT:    vpsrld $16, %ymm1, %ymm1450; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0451; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0452; AVX2-NEXT:    vpsrld $16, %ymm3, %ymm1453; AVX2-NEXT:    vpsrld $16, %ymm2, %ymm2454; AVX2-NEXT:    vpackusdw %ymm1, %ymm2, %ymm1455; AVX2-NEXT:    vpmulhw %ymm1, %ymm0, %ymm0456; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]457; AVX2-NEXT:    retq458;459; AVX512-LABEL: ashr_mulhuw_v16i16:460; AVX512:       # %bb.0:461; AVX512-NEXT:    vpsrld $16, %zmm0, %zmm0462; AVX512-NEXT:    vpsrld $16, %zmm1, %zmm1463; AVX512-NEXT:    vpmovdw %zmm1, %ymm1464; AVX512-NEXT:    vpmovdw %zmm0, %ymm0465; AVX512-NEXT:    vpmulhw %ymm1, %ymm0, %ymm0466; AVX512-NEXT:    retq467  %a1 = ashr <16 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>468  %b1 = ashr <16 x i32> %b, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>469  %c = mul <16 x i32> %a1, %b1470  %d = lshr <16 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>471  %e = trunc <16 x i32> %d to <16 x i16>472  ret <16 x i16> %e473}474 475define <16 x i16> @zext_mulhuw_v16i16_v16i48(<16 x i16> %a, <16 x i16> %b) {476; SSE-LABEL: zext_mulhuw_v16i16_v16i48:477; SSE:       # %bb.0:478; SSE-NEXT:    pmulhuw %xmm2, %xmm0479; SSE-NEXT:    pmulhuw %xmm3, %xmm1480; SSE-NEXT:    retq481;482; AVX-LABEL: zext_mulhuw_v16i16_v16i48:483; AVX:       # %bb.0:484; AVX-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0485; AVX-NEXT:    retq486  %a1 = zext <16 x i16> %a to <16 x i48>487  %b1 = zext <16 x i16> %b to <16 x i48>488  %c = mul <16 x i48> %a1, %b1489  %d = lshr <16 x i48> %c, <i48 16, i48 16, i48 16, i48 16, i48 16, i48 16, i48 16, i48 16, i48 16, i48 16, i48 16, i48 16, i48 16, i48 16, i48 16, i48 16>490  %e = trunc <16 x i48> %d to <16 x i16>491  ret <16 x i16> %e492}493 494define <32 x i16> @zext_mulhuw_v32i16(<32 x i16> %a, <32 x i16> %b) {495; SSE-LABEL: zext_mulhuw_v32i16:496; SSE:       # %bb.0:497; SSE-NEXT:    pmulhuw %xmm4, %xmm0498; SSE-NEXT:    pmulhuw %xmm5, %xmm1499; SSE-NEXT:    pmulhuw %xmm6, %xmm2500; SSE-NEXT:    pmulhuw %xmm7, %xmm3501; SSE-NEXT:    retq502;503; AVX2-LABEL: zext_mulhuw_v32i16:504; AVX2:       # %bb.0:505; AVX2-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm0506; AVX2-NEXT:    vpmulhuw %ymm3, %ymm1, %ymm1507; AVX2-NEXT:    retq508;509; AVX512F-LABEL: zext_mulhuw_v32i16:510; AVX512F:       # %bb.0:511; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2512; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3513; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm3, %ymm2514; AVX512F-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0515; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0516; AVX512F-NEXT:    retq517;518; AVX512BW-LABEL: zext_mulhuw_v32i16:519; AVX512BW:       # %bb.0:520; AVX512BW-NEXT:    vpmulhuw %zmm1, %zmm0, %zmm0521; AVX512BW-NEXT:    retq522  %a1 = zext <32 x i16> %a to <32 x i32>523  %b1 = zext <32 x i16> %b to <32 x i32>524  %c = mul <32 x i32> %a1, %b1525  %d = lshr <32 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>526  %e = trunc <32 x i32> %d to <32 x i16>527  ret <32 x i16> %e528}529 530define <32 x i16> @sext_mulhuw_v32i16(<32 x i16> %a, <32 x i16> %b) {531; SSE-LABEL: sext_mulhuw_v32i16:532; SSE:       # %bb.0:533; SSE-NEXT:    pmulhw %xmm4, %xmm0534; SSE-NEXT:    pmulhw %xmm5, %xmm1535; SSE-NEXT:    pmulhw %xmm6, %xmm2536; SSE-NEXT:    pmulhw %xmm7, %xmm3537; SSE-NEXT:    retq538;539; AVX2-LABEL: sext_mulhuw_v32i16:540; AVX2:       # %bb.0:541; AVX2-NEXT:    vpmulhw %ymm2, %ymm0, %ymm0542; AVX2-NEXT:    vpmulhw %ymm3, %ymm1, %ymm1543; AVX2-NEXT:    retq544;545; AVX512F-LABEL: sext_mulhuw_v32i16:546; AVX512F:       # %bb.0:547; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2548; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3549; AVX512F-NEXT:    vpmulhw %ymm2, %ymm3, %ymm2550; AVX512F-NEXT:    vpmulhw %ymm1, %ymm0, %ymm0551; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0552; AVX512F-NEXT:    retq553;554; AVX512BW-LABEL: sext_mulhuw_v32i16:555; AVX512BW:       # %bb.0:556; AVX512BW-NEXT:    vpmulhw %zmm1, %zmm0, %zmm0557; AVX512BW-NEXT:    retq558  %a1 = sext <32 x i16> %a to <32 x i32>559  %b1 = sext <32 x i16> %b to <32 x i32>560  %c = mul <32 x i32> %a1, %b1561  %d = lshr <32 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>562  %e = trunc <32 x i32> %d to <32 x i16>563  ret <32 x i16> %e564}565 566define <64 x i16> @zext_mulhuw_v64i16(<64 x i16> %a, <64 x i16> %b) {567; SSE-LABEL: zext_mulhuw_v64i16:568; SSE:       # %bb.0:569; SSE-NEXT:    movq %rdi, %rax570; SSE-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm0571; SSE-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm1572; SSE-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm2573; SSE-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm3574; SSE-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm4575; SSE-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm5576; SSE-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm6577; SSE-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm7578; SSE-NEXT:    movdqa %xmm7, 112(%rdi)579; SSE-NEXT:    movdqa %xmm6, 96(%rdi)580; SSE-NEXT:    movdqa %xmm5, 80(%rdi)581; SSE-NEXT:    movdqa %xmm4, 64(%rdi)582; SSE-NEXT:    movdqa %xmm3, 48(%rdi)583; SSE-NEXT:    movdqa %xmm2, 32(%rdi)584; SSE-NEXT:    movdqa %xmm1, 16(%rdi)585; SSE-NEXT:    movdqa %xmm0, (%rdi)586; SSE-NEXT:    retq587;588; AVX2-LABEL: zext_mulhuw_v64i16:589; AVX2:       # %bb.0:590; AVX2-NEXT:    vpmulhuw %ymm4, %ymm0, %ymm0591; AVX2-NEXT:    vpmulhuw %ymm5, %ymm1, %ymm1592; AVX2-NEXT:    vpmulhuw %ymm6, %ymm2, %ymm2593; AVX2-NEXT:    vpmulhuw %ymm7, %ymm3, %ymm3594; AVX2-NEXT:    retq595;596; AVX512F-LABEL: zext_mulhuw_v64i16:597; AVX512F:       # %bb.0:598; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm4599; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm5600; AVX512F-NEXT:    vpmulhuw %ymm4, %ymm5, %ymm4601; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm0602; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0603; AVX512F-NEXT:    vextracti64x4 $1, %zmm3, %ymm2604; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm4605; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm4, %ymm2606; AVX512F-NEXT:    vpmulhuw %ymm3, %ymm1, %ymm1607; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1608; AVX512F-NEXT:    retq609;610; AVX512BW-LABEL: zext_mulhuw_v64i16:611; AVX512BW:       # %bb.0:612; AVX512BW-NEXT:    vpmulhuw %zmm2, %zmm0, %zmm0613; AVX512BW-NEXT:    vpmulhuw %zmm3, %zmm1, %zmm1614; AVX512BW-NEXT:    retq615  %a1 = zext <64 x i16> %a to <64 x i32>616  %b1 = zext <64 x i16> %b to <64 x i32>617  %c = mul <64 x i32> %a1, %b1618  %d = lshr <64 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>619  %e = trunc <64 x i32> %d to <64 x i16>620  ret <64 x i16> %e621}622 623define <64 x i16> @sext_mulhuw_v64i16(<64 x i16> %a, <64 x i16> %b) {624; SSE-LABEL: sext_mulhuw_v64i16:625; SSE:       # %bb.0:626; SSE-NEXT:    movq %rdi, %rax627; SSE-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm0628; SSE-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm1629; SSE-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm2630; SSE-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm3631; SSE-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm4632; SSE-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm5633; SSE-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm6634; SSE-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm7635; SSE-NEXT:    movdqa %xmm7, 112(%rdi)636; SSE-NEXT:    movdqa %xmm6, 96(%rdi)637; SSE-NEXT:    movdqa %xmm5, 80(%rdi)638; SSE-NEXT:    movdqa %xmm4, 64(%rdi)639; SSE-NEXT:    movdqa %xmm3, 48(%rdi)640; SSE-NEXT:    movdqa %xmm2, 32(%rdi)641; SSE-NEXT:    movdqa %xmm1, 16(%rdi)642; SSE-NEXT:    movdqa %xmm0, (%rdi)643; SSE-NEXT:    retq644;645; AVX2-LABEL: sext_mulhuw_v64i16:646; AVX2:       # %bb.0:647; AVX2-NEXT:    vpmulhw %ymm4, %ymm0, %ymm0648; AVX2-NEXT:    vpmulhw %ymm5, %ymm1, %ymm1649; AVX2-NEXT:    vpmulhw %ymm6, %ymm2, %ymm2650; AVX2-NEXT:    vpmulhw %ymm7, %ymm3, %ymm3651; AVX2-NEXT:    retq652;653; AVX512F-LABEL: sext_mulhuw_v64i16:654; AVX512F:       # %bb.0:655; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm4656; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm5657; AVX512F-NEXT:    vpmulhw %ymm4, %ymm5, %ymm4658; AVX512F-NEXT:    vpmulhw %ymm2, %ymm0, %ymm0659; AVX512F-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0660; AVX512F-NEXT:    vextracti64x4 $1, %zmm3, %ymm2661; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm4662; AVX512F-NEXT:    vpmulhw %ymm2, %ymm4, %ymm2663; AVX512F-NEXT:    vpmulhw %ymm3, %ymm1, %ymm1664; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1665; AVX512F-NEXT:    retq666;667; AVX512BW-LABEL: sext_mulhuw_v64i16:668; AVX512BW:       # %bb.0:669; AVX512BW-NEXT:    vpmulhw %zmm2, %zmm0, %zmm0670; AVX512BW-NEXT:    vpmulhw %zmm3, %zmm1, %zmm1671; AVX512BW-NEXT:    retq672  %a1 = sext <64 x i16> %a to <64 x i32>673  %b1 = sext <64 x i16> %b to <64 x i32>674  %c = mul <64 x i32> %a1, %b1675  %d = lshr <64 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>676  %e = trunc <64 x i32> %d to <64 x i16>677  ret <64 x i16> %e678}679 680define <8 x i16> @zext_mulhuw_v8i16_i64(<8 x i16> %a, <8 x i16> %b) {681; SSE-LABEL: zext_mulhuw_v8i16_i64:682; SSE:       # %bb.0:683; SSE-NEXT:    pmulhuw %xmm1, %xmm0684; SSE-NEXT:    retq685;686; AVX-LABEL: zext_mulhuw_v8i16_i64:687; AVX:       # %bb.0:688; AVX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0689; AVX-NEXT:    retq690  %a1 = zext <8 x i16> %a to <8 x i64>691  %b1 = zext <8 x i16> %b to <8 x i64>692  %c = mul <8 x i64> %a1, %b1693  %d = lshr <8 x i64> %c, <i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16>694  %e = trunc <8 x i64> %d to <8 x i16>695  ret <8 x i16> %e696}697 698define <8 x i16> @sext_mulhuw_v8i16_i64(<8 x i16> %a, <8 x i16> %b) {699; SSE-LABEL: sext_mulhuw_v8i16_i64:700; SSE:       # %bb.0:701; SSE-NEXT:    pmulhw %xmm1, %xmm0702; SSE-NEXT:    retq703;704; AVX-LABEL: sext_mulhuw_v8i16_i64:705; AVX:       # %bb.0:706; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0707; AVX-NEXT:    retq708  %a1 = sext <8 x i16> %a to <8 x i64>709  %b1 = sext <8 x i16> %b to <8 x i64>710  %c = mul <8 x i64> %a1, %b1711  %d = lshr <8 x i64> %c, <i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16>712  %e = trunc <8 x i64> %d to <8 x i16>713  ret <8 x i16> %e714}715 716define <4 x i32> @zext_mulhuw_v4i16_lshr(<4 x i16> %a, <4 x i16> %b) {717; SSE2-LABEL: zext_mulhuw_v4i16_lshr:718; SSE2:       # %bb.0:719; SSE2-NEXT:    pmulhuw %xmm1, %xmm0720; SSE2-NEXT:    pxor %xmm1, %xmm1721; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]722; SSE2-NEXT:    retq723;724; SSE41-LABEL: zext_mulhuw_v4i16_lshr:725; SSE41:       # %bb.0:726; SSE41-NEXT:    pmulhuw %xmm1, %xmm0727; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero728; SSE41-NEXT:    retq729;730; AVX-LABEL: zext_mulhuw_v4i16_lshr:731; AVX:       # %bb.0:732; AVX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0733; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero734; AVX-NEXT:    retq735  %a1 = zext <4 x i16> %a to <4 x i32>736  %b1 = zext <4 x i16> %b to <4 x i32>737  %c = mul <4 x i32> %a1, %b1738  %d = lshr <4 x i32> %c, <i32 16, i32 16, i32 16, i32 16>739  ret <4 x i32> %d740}741 742define <4 x i32> @mulhsw_v4i16_lshr(<4 x i16> %a, <4 x i16> %b) {743; SSE2-LABEL: mulhsw_v4i16_lshr:744; SSE2:       # %bb.0:745; SSE2-NEXT:    pmulhw %xmm1, %xmm0746; SSE2-NEXT:    pxor %xmm1, %xmm1747; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]748; SSE2-NEXT:    retq749;750; SSE41-LABEL: mulhsw_v4i16_lshr:751; SSE41:       # %bb.0:752; SSE41-NEXT:    pmulhw %xmm1, %xmm0753; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero754; SSE41-NEXT:    retq755;756; AVX-LABEL: mulhsw_v4i16_lshr:757; AVX:       # %bb.0:758; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0759; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero760; AVX-NEXT:    retq761  %a1 = sext <4 x i16> %a to <4 x i32>762  %b1 = sext <4 x i16> %b to <4 x i32>763  %c = mul <4 x i32> %a1, %b1764  %d = lshr <4 x i32> %c, <i32 16, i32 16, i32 16, i32 16>765  ret <4 x i32> %d766}767 768define <4 x i32> @mulhsw_v4i16_ashr(<4 x i16> %a, <4 x i16> %b) {769; SSE2-LABEL: mulhsw_v4i16_ashr:770; SSE2:       # %bb.0:771; SSE2-NEXT:    pmulhw %xmm1, %xmm0772; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]773; SSE2-NEXT:    psrad $16, %xmm0774; SSE2-NEXT:    retq775;776; SSE41-LABEL: mulhsw_v4i16_ashr:777; SSE41:       # %bb.0:778; SSE41-NEXT:    pmulhw %xmm1, %xmm0779; SSE41-NEXT:    pmovsxwd %xmm0, %xmm0780; SSE41-NEXT:    retq781;782; AVX-LABEL: mulhsw_v4i16_ashr:783; AVX:       # %bb.0:784; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0785; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0786; AVX-NEXT:    retq787  %a1 = sext <4 x i16> %a to <4 x i32>788  %b1 = sext <4 x i16> %b to <4 x i32>789  %c = mul <4 x i32> %a1, %b1790  %d = ashr <4 x i32> %c, <i32 16, i32 16, i32 16, i32 16>791  ret <4 x i32> %d792}793 794define <8 x i32> @zext_mulhuw_v8i16_lshr(<8 x i16> %a, <8 x i16> %b) {795; SSE2-LABEL: zext_mulhuw_v8i16_lshr:796; SSE2:       # %bb.0:797; SSE2-NEXT:    pmulhuw %xmm0, %xmm1798; SSE2-NEXT:    pxor %xmm2, %xmm2799; SSE2-NEXT:    movdqa %xmm1, %xmm0800; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]801; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]802; SSE2-NEXT:    retq803;804; SSE41-LABEL: zext_mulhuw_v8i16_lshr:805; SSE41:       # %bb.0:806; SSE41-NEXT:    movdqa %xmm0, %xmm2807; SSE41-NEXT:    pmulhuw %xmm1, %xmm2808; SSE41-NEXT:    pxor %xmm1, %xmm1809; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero810; SSE41-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]811; SSE41-NEXT:    movdqa %xmm2, %xmm1812; SSE41-NEXT:    retq813;814; AVX-LABEL: zext_mulhuw_v8i16_lshr:815; AVX:       # %bb.0:816; AVX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0817; AVX-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero818; AVX-NEXT:    retq819  %a1 = zext <8 x i16> %a to <8 x i32>820  %b1 = zext <8 x i16> %b to <8 x i32>821  %c = mul <8 x i32> %a1, %b1822  %d = lshr <8 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>823  ret <8 x i32> %d824}825 826define <8 x i32> @mulhsw_v8i16_lshr(<8 x i16> %a, <8 x i16> %b) {827; SSE2-LABEL: mulhsw_v8i16_lshr:828; SSE2:       # %bb.0:829; SSE2-NEXT:    pmulhw %xmm0, %xmm1830; SSE2-NEXT:    pxor %xmm2, %xmm2831; SSE2-NEXT:    movdqa %xmm1, %xmm0832; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]833; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]834; SSE2-NEXT:    retq835;836; SSE41-LABEL: mulhsw_v8i16_lshr:837; SSE41:       # %bb.0:838; SSE41-NEXT:    movdqa %xmm0, %xmm2839; SSE41-NEXT:    pmulhw %xmm1, %xmm2840; SSE41-NEXT:    pxor %xmm1, %xmm1841; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero842; SSE41-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]843; SSE41-NEXT:    movdqa %xmm2, %xmm1844; SSE41-NEXT:    retq845;846; AVX-LABEL: mulhsw_v8i16_lshr:847; AVX:       # %bb.0:848; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0849; AVX-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero850; AVX-NEXT:    retq851  %a1 = sext <8 x i16> %a to <8 x i32>852  %b1 = sext <8 x i16> %b to <8 x i32>853  %c = mul <8 x i32> %a1, %b1854  %d = lshr <8 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>855  ret <8 x i32> %d856}857 858define <8 x i32> @mulhsw_v8i16_ashr(<8 x i16> %a, <8 x i16> %b) {859; SSE2-LABEL: mulhsw_v8i16_ashr:860; SSE2:       # %bb.0:861; SSE2-NEXT:    pmulhw %xmm1, %xmm0862; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]863; SSE2-NEXT:    psrad $16, %xmm2864; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]865; SSE2-NEXT:    psrad $16, %xmm1866; SSE2-NEXT:    movdqa %xmm2, %xmm0867; SSE2-NEXT:    retq868;869; SSE41-LABEL: mulhsw_v8i16_ashr:870; SSE41:       # %bb.0:871; SSE41-NEXT:    pmulhw %xmm1, %xmm0872; SSE41-NEXT:    pmovsxwd %xmm0, %xmm2873; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]874; SSE41-NEXT:    pmovsxwd %xmm0, %xmm1875; SSE41-NEXT:    movdqa %xmm2, %xmm0876; SSE41-NEXT:    retq877;878; AVX-LABEL: mulhsw_v8i16_ashr:879; AVX:       # %bb.0:880; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0881; AVX-NEXT:    vpmovsxwd %xmm0, %ymm0882; AVX-NEXT:    retq883  %a1 = sext <8 x i16> %a to <8 x i32>884  %b1 = sext <8 x i16> %b to <8 x i32>885  %c = mul <8 x i32> %a1, %b1886  %d = ashr <8 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>887  ret <8 x i32> %d888}889 890define <16 x i32> @zext_mulhuw_v16i16_lshr(<16 x i16> %a, <16 x i16> %b) {891; SSE2-LABEL: zext_mulhuw_v16i16_lshr:892; SSE2:       # %bb.0:893; SSE2-NEXT:    movdqa %xmm0, %xmm4894; SSE2-NEXT:    pmulhuw %xmm2, %xmm4895; SSE2-NEXT:    pxor %xmm5, %xmm5896; SSE2-NEXT:    movdqa %xmm4, %xmm0897; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]898; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]899; SSE2-NEXT:    pmulhuw %xmm1, %xmm3900; SSE2-NEXT:    movdqa %xmm3, %xmm2901; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]902; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]903; SSE2-NEXT:    movdqa %xmm4, %xmm1904; SSE2-NEXT:    retq905;906; SSE41-LABEL: zext_mulhuw_v16i16_lshr:907; SSE41:       # %bb.0:908; SSE41-NEXT:    movdqa %xmm1, %xmm4909; SSE41-NEXT:    movdqa %xmm0, %xmm1910; SSE41-NEXT:    pmulhuw %xmm2, %xmm1911; SSE41-NEXT:    pxor %xmm5, %xmm5912; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero913; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]914; SSE41-NEXT:    pmulhuw %xmm3, %xmm4915; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero916; SSE41-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]917; SSE41-NEXT:    movdqa %xmm4, %xmm3918; SSE41-NEXT:    retq919;920; AVX2-LABEL: zext_mulhuw_v16i16_lshr:921; AVX2:       # %bb.0:922; AVX2-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm1923; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero924; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1925; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero926; AVX2-NEXT:    retq927;928; AVX512-LABEL: zext_mulhuw_v16i16_lshr:929; AVX512:       # %bb.0:930; AVX512-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0931; AVX512-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero932; AVX512-NEXT:    retq933  %a1 = zext <16 x i16> %a to <16 x i32>934  %b1 = zext <16 x i16> %b to <16 x i32>935  %c = mul <16 x i32> %a1, %b1936  %d = lshr <16 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>937  ret <16 x i32> %d938}939 940; PR109790941define void @PR109790(ptr sret([32 x i8]) %ret, ptr %a) {942; SSE-LABEL: PR109790:943; SSE:       # %bb.0:944; SSE-NEXT:    movq %rdi, %rax945; SSE-NEXT:    movdqa {{.*#+}} xmm0 = [32767,32767,32767,32767,32767,32767,32767,32767]946; SSE-NEXT:    movdqa (%rsi), %xmm1947; SSE-NEXT:    pand %xmm0, %xmm1948; SSE-NEXT:    pand 16(%rsi), %xmm0949; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [64536,64536,64536,64536,64536,64536,64536,64536]950; SSE-NEXT:    pmulhw %xmm2, %xmm0951; SSE-NEXT:    pmulhw %xmm2, %xmm1952; SSE-NEXT:    movdqa %xmm1, (%rdi)953; SSE-NEXT:    movdqa %xmm0, 16(%rdi)954; SSE-NEXT:    retq955;956; AVX2-LABEL: PR109790:957; AVX2:       # %bb.0:958; AVX2-NEXT:    movq %rdi, %rax959; AVX2-NEXT:    vmovdqa (%rsi), %ymm0960; AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0961; AVX2-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [64536,64536,64536,64536,64536,64536,64536,64536,64536,64536,64536,64536,64536,64536,64536,64536]962; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)963; AVX2-NEXT:    vzeroupper964; AVX2-NEXT:    retq965;966; AVX512F-LABEL: PR109790:967; AVX512F:       # %bb.0:968; AVX512F-NEXT:    movq %rdi, %rax969; AVX512F-NEXT:    vmovdqa (%rsi), %ymm0970; AVX512F-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0971; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero972; AVX512F-NEXT:    vpmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0973; AVX512F-NEXT:    vpsrld $16, %zmm0, %zmm0974; AVX512F-NEXT:    vpmovdw %zmm0, (%rdi)975; AVX512F-NEXT:    vzeroupper976; AVX512F-NEXT:    retq977;978; AVX512BW-LABEL: PR109790:979; AVX512BW:       # %bb.0:980; AVX512BW-NEXT:    movq %rdi, %rax981; AVX512BW-NEXT:    vmovdqa (%rsi), %ymm0982; AVX512BW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0983; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero984; AVX512BW-NEXT:    vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [64536,0,64536,0,64536,0,64536,0,64536,0,64536,0,64536,0,64536,0,64536,0,64536,0,64536,0,64536,0,64536,0,64536,0,64536,0,64536,0]985; AVX512BW-NEXT:    vpsrld $16, %zmm0, %zmm0986; AVX512BW-NEXT:    vpmovdw %zmm0, (%rdi)987; AVX512BW-NEXT:    vzeroupper988; AVX512BW-NEXT:    retq989  %load = load <16 x i16>, ptr %a, align 32990  %and = and <16 x i16> %load, <i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767>991  %ext = zext nneg <16 x i16> %and to <16 x i32>992  %mul = mul nsw <16 x i32> %ext, <i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000>993  %srl = lshr <16 x i32> %mul, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>994  %res = trunc nuw <16 x i32> %srl to <16 x i16>995  store <16 x i16> %res, ptr %ret, align 32996  ret void997}998 999; PR1097901000define <16 x i16> @zext_mulhuw_v16i16_negative_constant(<16 x i16> %a) {1001; SSE-LABEL: zext_mulhuw_v16i16_negative_constant:1002; SSE:       # %bb.0:1003; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32767,32767,32767,32767,32767,32767,32767,32767]1004; SSE-NEXT:    pand %xmm2, %xmm11005; SSE-NEXT:    pand %xmm2, %xmm01006; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [64536,64536,64536,64536,64536,64536,64536,64536]1007; SSE-NEXT:    pmulhw %xmm2, %xmm01008; SSE-NEXT:    pmulhw %xmm2, %xmm11009; SSE-NEXT:    retq1010;1011; AVX-LABEL: zext_mulhuw_v16i16_negative_constant:1012; AVX:       # %bb.0:1013; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01014; AVX-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [64536,64536,64536,64536,64536,64536,64536,64536,64536,64536,64536,64536,64536,64536,64536,64536]1015; AVX-NEXT:    retq1016  %k = and <16 x i16> %a, <i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767>1017  %x = zext nneg <16 x i16> %k to <16 x i32>1018  %m = mul nsw <16 x i32> %x, <i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000, i32 -1000>1019  %s = lshr <16 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1020  %t = trunc nuw <16 x i32> %s to <16 x i16>1021  ret <16 x i16> %t1022}1023 1024; PR1097901025define <16 x i16> @zext_mulhuw_v16i16_positive_constant(<16 x i16> %a) {1026; SSE-LABEL: zext_mulhuw_v16i16_positive_constant:1027; SSE:       # %bb.0:1028; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32767,32767,32767,32767,32767,32767,32767,32767]1029; SSE-NEXT:    pand %xmm2, %xmm11030; SSE-NEXT:    pand %xmm2, %xmm01031; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1000,1000,1000,1000,1000,1000,1000,1000]1032; SSE-NEXT:    pmulhw %xmm2, %xmm01033; SSE-NEXT:    pmulhw %xmm2, %xmm11034; SSE-NEXT:    retq1035;1036; AVX-LABEL: zext_mulhuw_v16i16_positive_constant:1037; AVX:       # %bb.0:1038; AVX-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01039; AVX-NEXT:    vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000]1040; AVX-NEXT:    retq1041  %k = and <16 x i16> %a, <i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767>1042  %x = zext nneg <16 x i16> %k to <16 x i32>1043  %m = mul nuw nsw <16 x i32> %x, <i32 1000, i32 1000, i32 1000, i32 1000, i32 1000, i32 1000, i32 1000, i32 1000, i32 1000, i32 1000, i32 1000, i32 1000, i32 1000, i32 1000, i32 1000, i32 1000>1044  %s = lshr <16 x i32> %m, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1045  %t = trunc nuw nsw <16 x i32> %s to <16 x i16>1046  ret <16 x i16> %t1047}1048 1049define <16 x i32> @mulhsw_v16i16_lshr(<16 x i16> %a, <16 x i16> %b) {1050; SSE2-LABEL: mulhsw_v16i16_lshr:1051; SSE2:       # %bb.0:1052; SSE2-NEXT:    movdqa %xmm0, %xmm41053; SSE2-NEXT:    pmulhw %xmm2, %xmm41054; SSE2-NEXT:    pxor %xmm5, %xmm51055; SSE2-NEXT:    movdqa %xmm4, %xmm01056; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]1057; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]1058; SSE2-NEXT:    pmulhw %xmm1, %xmm31059; SSE2-NEXT:    movdqa %xmm3, %xmm21060; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]1061; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]1062; SSE2-NEXT:    movdqa %xmm4, %xmm11063; SSE2-NEXT:    retq1064;1065; SSE41-LABEL: mulhsw_v16i16_lshr:1066; SSE41:       # %bb.0:1067; SSE41-NEXT:    movdqa %xmm1, %xmm41068; SSE41-NEXT:    movdqa %xmm0, %xmm11069; SSE41-NEXT:    pmulhw %xmm2, %xmm11070; SSE41-NEXT:    pxor %xmm5, %xmm51071; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero1072; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]1073; SSE41-NEXT:    pmulhw %xmm3, %xmm41074; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero1075; SSE41-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]1076; SSE41-NEXT:    movdqa %xmm4, %xmm31077; SSE41-NEXT:    retq1078;1079; AVX2-LABEL: mulhsw_v16i16_lshr:1080; AVX2:       # %bb.0:1081; AVX2-NEXT:    vpmulhw %ymm1, %ymm0, %ymm11082; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1083; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm11084; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1085; AVX2-NEXT:    retq1086;1087; AVX512-LABEL: mulhsw_v16i16_lshr:1088; AVX512:       # %bb.0:1089; AVX512-NEXT:    vpmulhw %ymm1, %ymm0, %ymm01090; AVX512-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero1091; AVX512-NEXT:    retq1092  %a1 = sext <16 x i16> %a to <16 x i32>1093  %b1 = sext <16 x i16> %b to <16 x i32>1094  %c = mul <16 x i32> %a1, %b11095  %d = lshr <16 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1096  ret <16 x i32> %d1097}1098 1099define <16 x i32> @mulhsw_v16i16_ashr(<16 x i16> %a, <16 x i16> %b) {1100; SSE2-LABEL: mulhsw_v16i16_ashr:1101; SSE2:       # %bb.0:1102; SSE2-NEXT:    pmulhw %xmm2, %xmm01103; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]1104; SSE2-NEXT:    psrad $16, %xmm51105; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]1106; SSE2-NEXT:    psrad $16, %xmm41107; SSE2-NEXT:    pmulhw %xmm3, %xmm11108; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]1109; SSE2-NEXT:    psrad $16, %xmm21110; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]1111; SSE2-NEXT:    psrad $16, %xmm31112; SSE2-NEXT:    movdqa %xmm5, %xmm01113; SSE2-NEXT:    movdqa %xmm4, %xmm11114; SSE2-NEXT:    retq1115;1116; SSE41-LABEL: mulhsw_v16i16_ashr:1117; SSE41:       # %bb.0:1118; SSE41-NEXT:    pmulhw %xmm2, %xmm01119; SSE41-NEXT:    pmovsxwd %xmm0, %xmm41120; SSE41-NEXT:    pmulhw %xmm3, %xmm11121; SSE41-NEXT:    pmovsxwd %xmm1, %xmm21122; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]1123; SSE41-NEXT:    pmovsxwd %xmm0, %xmm51124; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]1125; SSE41-NEXT:    pmovsxwd %xmm0, %xmm31126; SSE41-NEXT:    movdqa %xmm4, %xmm01127; SSE41-NEXT:    movdqa %xmm5, %xmm11128; SSE41-NEXT:    retq1129;1130; AVX2-LABEL: mulhsw_v16i16_ashr:1131; AVX2:       # %bb.0:1132; AVX2-NEXT:    vpmulhw %ymm1, %ymm0, %ymm11133; AVX2-NEXT:    vpmovsxwd %xmm1, %ymm01134; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm11135; AVX2-NEXT:    vpmovsxwd %xmm1, %ymm11136; AVX2-NEXT:    retq1137;1138; AVX512-LABEL: mulhsw_v16i16_ashr:1139; AVX512:       # %bb.0:1140; AVX512-NEXT:    vpmulhw %ymm1, %ymm0, %ymm01141; AVX512-NEXT:    vpmovsxwd %ymm0, %zmm01142; AVX512-NEXT:    retq1143  %a1 = sext <16 x i16> %a to <16 x i32>1144  %b1 = sext <16 x i16> %b to <16 x i32>1145  %c = mul <16 x i32> %a1, %b11146  %d = ashr <16 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1147  ret <16 x i32> %d1148}1149 1150define <32 x i32> @zext_mulhuw_v32i16_lshr(<32 x i16> %a, <32 x i16> %b) {1151; SSE2-LABEL: zext_mulhuw_v32i16_lshr:1152; SSE2:       # %bb.0:1153; SSE2-NEXT:    movq %rdi, %rax1154; SSE2-NEXT:    pmulhuw %xmm4, %xmm01155; SSE2-NEXT:    pxor %xmm4, %xmm41156; SSE2-NEXT:    movdqa %xmm0, %xmm81157; SSE2-NEXT:    punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3]1158; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]1159; SSE2-NEXT:    pmulhuw %xmm5, %xmm11160; SSE2-NEXT:    movdqa %xmm1, %xmm51161; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]1162; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]1163; SSE2-NEXT:    pmulhuw %xmm6, %xmm21164; SSE2-NEXT:    movdqa %xmm2, %xmm61165; SSE2-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]1166; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]1167; SSE2-NEXT:    pmulhuw %xmm7, %xmm31168; SSE2-NEXT:    movdqa %xmm3, %xmm71169; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3]1170; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]1171; SSE2-NEXT:    movdqa %xmm3, 112(%rdi)1172; SSE2-NEXT:    movdqa %xmm7, 96(%rdi)1173; SSE2-NEXT:    movdqa %xmm2, 80(%rdi)1174; SSE2-NEXT:    movdqa %xmm6, 64(%rdi)1175; SSE2-NEXT:    movdqa %xmm1, 48(%rdi)1176; SSE2-NEXT:    movdqa %xmm5, 32(%rdi)1177; SSE2-NEXT:    movdqa %xmm0, 16(%rdi)1178; SSE2-NEXT:    movdqa %xmm8, (%rdi)1179; SSE2-NEXT:    retq1180;1181; SSE41-LABEL: zext_mulhuw_v32i16_lshr:1182; SSE41:       # %bb.0:1183; SSE41-NEXT:    movq %rdi, %rax1184; SSE41-NEXT:    pmulhuw %xmm4, %xmm01185; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero1186; SSE41-NEXT:    pxor %xmm8, %xmm81187; SSE41-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7]1188; SSE41-NEXT:    pmulhuw %xmm5, %xmm11189; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero1190; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]1191; SSE41-NEXT:    pmulhuw %xmm6, %xmm21192; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero1193; SSE41-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]1194; SSE41-NEXT:    pmulhuw %xmm7, %xmm31195; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm7 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero1196; SSE41-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm8[4],xmm3[5],xmm8[5],xmm3[6],xmm8[6],xmm3[7],xmm8[7]1197; SSE41-NEXT:    movdqa %xmm3, 112(%rdi)1198; SSE41-NEXT:    movdqa %xmm7, 96(%rdi)1199; SSE41-NEXT:    movdqa %xmm2, 80(%rdi)1200; SSE41-NEXT:    movdqa %xmm6, 64(%rdi)1201; SSE41-NEXT:    movdqa %xmm1, 48(%rdi)1202; SSE41-NEXT:    movdqa %xmm5, 32(%rdi)1203; SSE41-NEXT:    movdqa %xmm0, 16(%rdi)1204; SSE41-NEXT:    movdqa %xmm4, (%rdi)1205; SSE41-NEXT:    retq1206;1207; AVX2-LABEL: zext_mulhuw_v32i16_lshr:1208; AVX2:       # %bb.0:1209; AVX2-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm21210; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero1211; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm21212; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero1213; AVX2-NEXT:    vpmulhuw %ymm3, %ymm1, %ymm11214; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1215; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm11216; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1217; AVX2-NEXT:    vmovdqa %ymm4, %ymm11218; AVX2-NEXT:    retq1219;1220; AVX512F-LABEL: zext_mulhuw_v32i16_lshr:1221; AVX512F:       # %bb.0:1222; AVX512F-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm21223; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero1224; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm11225; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm01226; AVX512F-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm01227; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero1228; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm01229; AVX512F-NEXT:    retq1230;1231; AVX512BW-LABEL: zext_mulhuw_v32i16_lshr:1232; AVX512BW:       # %bb.0:1233; AVX512BW-NEXT:    vpmulhuw %zmm1, %zmm0, %zmm11234; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero1235; AVX512BW-NEXT:    vextracti64x4 $1, %zmm1, %ymm11236; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero1237; AVX512BW-NEXT:    retq1238  %a1 = zext <32 x i16> %a to <32 x i32>1239  %b1 = zext <32 x i16> %b to <32 x i32>1240  %c = mul <32 x i32> %a1, %b11241  %d = lshr <32 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1242  ret <32 x i32> %d1243}1244 1245define <32 x i32> @mulhsw_v32i16_lshr(<32 x i16> %a, <32 x i16> %b) {1246; SSE2-LABEL: mulhsw_v32i16_lshr:1247; SSE2:       # %bb.0:1248; SSE2-NEXT:    movq %rdi, %rax1249; SSE2-NEXT:    pmulhw %xmm4, %xmm01250; SSE2-NEXT:    pxor %xmm4, %xmm41251; SSE2-NEXT:    movdqa %xmm0, %xmm81252; SSE2-NEXT:    punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3]1253; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]1254; SSE2-NEXT:    pmulhw %xmm5, %xmm11255; SSE2-NEXT:    movdqa %xmm1, %xmm51256; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]1257; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]1258; SSE2-NEXT:    pmulhw %xmm6, %xmm21259; SSE2-NEXT:    movdqa %xmm2, %xmm61260; SSE2-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]1261; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]1262; SSE2-NEXT:    pmulhw %xmm7, %xmm31263; SSE2-NEXT:    movdqa %xmm3, %xmm71264; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3]1265; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]1266; SSE2-NEXT:    movdqa %xmm3, 112(%rdi)1267; SSE2-NEXT:    movdqa %xmm7, 96(%rdi)1268; SSE2-NEXT:    movdqa %xmm2, 80(%rdi)1269; SSE2-NEXT:    movdqa %xmm6, 64(%rdi)1270; SSE2-NEXT:    movdqa %xmm1, 48(%rdi)1271; SSE2-NEXT:    movdqa %xmm5, 32(%rdi)1272; SSE2-NEXT:    movdqa %xmm0, 16(%rdi)1273; SSE2-NEXT:    movdqa %xmm8, (%rdi)1274; SSE2-NEXT:    retq1275;1276; SSE41-LABEL: mulhsw_v32i16_lshr:1277; SSE41:       # %bb.0:1278; SSE41-NEXT:    movq %rdi, %rax1279; SSE41-NEXT:    pmulhw %xmm4, %xmm01280; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero1281; SSE41-NEXT:    pxor %xmm8, %xmm81282; SSE41-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7]1283; SSE41-NEXT:    pmulhw %xmm5, %xmm11284; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero1285; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7]1286; SSE41-NEXT:    pmulhw %xmm6, %xmm21287; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero1288; SSE41-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]1289; SSE41-NEXT:    pmulhw %xmm7, %xmm31290; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm7 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero1291; SSE41-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm8[4],xmm3[5],xmm8[5],xmm3[6],xmm8[6],xmm3[7],xmm8[7]1292; SSE41-NEXT:    movdqa %xmm3, 112(%rdi)1293; SSE41-NEXT:    movdqa %xmm7, 96(%rdi)1294; SSE41-NEXT:    movdqa %xmm2, 80(%rdi)1295; SSE41-NEXT:    movdqa %xmm6, 64(%rdi)1296; SSE41-NEXT:    movdqa %xmm1, 48(%rdi)1297; SSE41-NEXT:    movdqa %xmm5, 32(%rdi)1298; SSE41-NEXT:    movdqa %xmm0, 16(%rdi)1299; SSE41-NEXT:    movdqa %xmm4, (%rdi)1300; SSE41-NEXT:    retq1301;1302; AVX2-LABEL: mulhsw_v32i16_lshr:1303; AVX2:       # %bb.0:1304; AVX2-NEXT:    vpmulhw %ymm2, %ymm0, %ymm21305; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero1306; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm21307; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero1308; AVX2-NEXT:    vpmulhw %ymm3, %ymm1, %ymm11309; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1310; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm11311; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1312; AVX2-NEXT:    vmovdqa %ymm4, %ymm11313; AVX2-NEXT:    retq1314;1315; AVX512F-LABEL: mulhsw_v32i16_lshr:1316; AVX512F:       # %bb.0:1317; AVX512F-NEXT:    vpmulhw %ymm1, %ymm0, %ymm21318; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero1319; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm11320; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm01321; AVX512F-NEXT:    vpmulhw %ymm1, %ymm0, %ymm01322; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero1323; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm01324; AVX512F-NEXT:    retq1325;1326; AVX512BW-LABEL: mulhsw_v32i16_lshr:1327; AVX512BW:       # %bb.0:1328; AVX512BW-NEXT:    vpmulhw %zmm1, %zmm0, %zmm11329; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero1330; AVX512BW-NEXT:    vextracti64x4 $1, %zmm1, %ymm11331; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero1332; AVX512BW-NEXT:    retq1333  %a1 = sext <32 x i16> %a to <32 x i32>1334  %b1 = sext <32 x i16> %b to <32 x i32>1335  %c = mul <32 x i32> %a1, %b11336  %d = lshr <32 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1337  ret <32 x i32> %d1338}1339 1340define <32 x i32> @mulhsw_v32i16_ashr(<32 x i16> %a, <32 x i16> %b) {1341; SSE2-LABEL: mulhsw_v32i16_ashr:1342; SSE2:       # %bb.0:1343; SSE2-NEXT:    movq %rdi, %rax1344; SSE2-NEXT:    pmulhw %xmm4, %xmm01345; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]1346; SSE2-NEXT:    psrad $16, %xmm41347; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]1348; SSE2-NEXT:    psrad $16, %xmm01349; SSE2-NEXT:    pmulhw %xmm5, %xmm11350; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]1351; SSE2-NEXT:    psrad $16, %xmm51352; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]1353; SSE2-NEXT:    psrad $16, %xmm11354; SSE2-NEXT:    pmulhw %xmm6, %xmm21355; SSE2-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]1356; SSE2-NEXT:    psrad $16, %xmm61357; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]1358; SSE2-NEXT:    psrad $16, %xmm21359; SSE2-NEXT:    pmulhw %xmm7, %xmm31360; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]1361; SSE2-NEXT:    psrad $16, %xmm71362; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]1363; SSE2-NEXT:    psrad $16, %xmm31364; SSE2-NEXT:    movdqa %xmm3, 112(%rdi)1365; SSE2-NEXT:    movdqa %xmm7, 96(%rdi)1366; SSE2-NEXT:    movdqa %xmm2, 80(%rdi)1367; SSE2-NEXT:    movdqa %xmm6, 64(%rdi)1368; SSE2-NEXT:    movdqa %xmm1, 48(%rdi)1369; SSE2-NEXT:    movdqa %xmm5, 32(%rdi)1370; SSE2-NEXT:    movdqa %xmm0, 16(%rdi)1371; SSE2-NEXT:    movdqa %xmm4, (%rdi)1372; SSE2-NEXT:    retq1373;1374; SSE41-LABEL: mulhsw_v32i16_ashr:1375; SSE41:       # %bb.0:1376; SSE41-NEXT:    movq %rdi, %rax1377; SSE41-NEXT:    pmulhw %xmm4, %xmm01378; SSE41-NEXT:    pmovsxwd %xmm0, %xmm41379; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]1380; SSE41-NEXT:    pmovsxwd %xmm0, %xmm01381; SSE41-NEXT:    pmulhw %xmm5, %xmm11382; SSE41-NEXT:    pmovsxwd %xmm1, %xmm51383; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]1384; SSE41-NEXT:    pmovsxwd %xmm1, %xmm11385; SSE41-NEXT:    pmulhw %xmm6, %xmm21386; SSE41-NEXT:    pmovsxwd %xmm2, %xmm61387; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]1388; SSE41-NEXT:    pmovsxwd %xmm2, %xmm21389; SSE41-NEXT:    pmulhw %xmm7, %xmm31390; SSE41-NEXT:    pmovsxwd %xmm3, %xmm71391; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]1392; SSE41-NEXT:    pmovsxwd %xmm3, %xmm31393; SSE41-NEXT:    movdqa %xmm3, 112(%rdi)1394; SSE41-NEXT:    movdqa %xmm7, 96(%rdi)1395; SSE41-NEXT:    movdqa %xmm2, 80(%rdi)1396; SSE41-NEXT:    movdqa %xmm6, 64(%rdi)1397; SSE41-NEXT:    movdqa %xmm1, 48(%rdi)1398; SSE41-NEXT:    movdqa %xmm5, 32(%rdi)1399; SSE41-NEXT:    movdqa %xmm0, 16(%rdi)1400; SSE41-NEXT:    movdqa %xmm4, (%rdi)1401; SSE41-NEXT:    retq1402;1403; AVX2-LABEL: mulhsw_v32i16_ashr:1404; AVX2:       # %bb.0:1405; AVX2-NEXT:    vpmulhw %ymm2, %ymm0, %ymm21406; AVX2-NEXT:    vpmovsxwd %xmm2, %ymm01407; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm21408; AVX2-NEXT:    vpmovsxwd %xmm2, %ymm41409; AVX2-NEXT:    vpmulhw %ymm3, %ymm1, %ymm11410; AVX2-NEXT:    vpmovsxwd %xmm1, %ymm21411; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm11412; AVX2-NEXT:    vpmovsxwd %xmm1, %ymm31413; AVX2-NEXT:    vmovdqa %ymm4, %ymm11414; AVX2-NEXT:    retq1415;1416; AVX512F-LABEL: mulhsw_v32i16_ashr:1417; AVX512F:       # %bb.0:1418; AVX512F-NEXT:    vpmulhw %ymm1, %ymm0, %ymm21419; AVX512F-NEXT:    vpmovsxwd %ymm2, %zmm21420; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm11421; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm01422; AVX512F-NEXT:    vpmulhw %ymm1, %ymm0, %ymm01423; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm11424; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm01425; AVX512F-NEXT:    retq1426;1427; AVX512BW-LABEL: mulhsw_v32i16_ashr:1428; AVX512BW:       # %bb.0:1429; AVX512BW-NEXT:    vpmulhw %zmm1, %zmm0, %zmm11430; AVX512BW-NEXT:    vpmovsxwd %ymm1, %zmm01431; AVX512BW-NEXT:    vextracti64x4 $1, %zmm1, %ymm11432; AVX512BW-NEXT:    vpmovsxwd %ymm1, %zmm11433; AVX512BW-NEXT:    retq1434  %a1 = sext <32 x i16> %a to <32 x i32>1435  %b1 = sext <32 x i16> %b to <32 x i32>1436  %c = mul <32 x i32> %a1, %b11437  %d = ashr <32 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1438  ret <32 x i32> %d1439}1440 1441define <64 x i32> @zext_mulhuw_v64i16_lshr(<64 x i16> %a, <64 x i16> %b) {1442; SSE2-LABEL: zext_mulhuw_v64i16_lshr:1443; SSE2:       # %bb.0:1444; SSE2-NEXT:    movdqa %xmm7, %xmm81445; SSE2-NEXT:    movq %rdi, %rax1446; SSE2-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm01447; SSE2-NEXT:    pxor %xmm10, %xmm101448; SSE2-NEXT:    movdqa %xmm0, %xmm71449; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm10[0],xmm7[1],xmm10[1],xmm7[2],xmm10[2],xmm7[3],xmm10[3]1450; SSE2-NEXT:    movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1451; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]1452; SSE2-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm11453; SSE2-NEXT:    movdqa %xmm1, %xmm91454; SSE2-NEXT:    punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]1455; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]1456; SSE2-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm21457; SSE2-NEXT:    movdqa %xmm2, %xmm111458; SSE2-NEXT:    punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]1459; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm10[4],xmm2[5],xmm10[5],xmm2[6],xmm10[6],xmm2[7],xmm10[7]1460; SSE2-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm31461; SSE2-NEXT:    movdqa %xmm3, %xmm121462; SSE2-NEXT:    punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]1463; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]1464; SSE2-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm41465; SSE2-NEXT:    movdqa %xmm4, %xmm131466; SSE2-NEXT:    punpcklwd {{.*#+}} xmm13 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3]1467; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm10[4],xmm4[5],xmm10[5],xmm4[6],xmm10[6],xmm4[7],xmm10[7]1468; SSE2-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm51469; SSE2-NEXT:    movdqa %xmm5, %xmm141470; SSE2-NEXT:    punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]1471; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm10[4],xmm5[5],xmm10[5],xmm5[6],xmm10[6],xmm5[7],xmm10[7]1472; SSE2-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm61473; SSE2-NEXT:    movdqa %xmm6, %xmm151474; SSE2-NEXT:    punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm10[0],xmm15[1],xmm10[1],xmm15[2],xmm10[2],xmm15[3],xmm10[3]1475; SSE2-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm10[4],xmm6[5],xmm10[5],xmm6[6],xmm10[6],xmm6[7],xmm10[7]1476; SSE2-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm81477; SSE2-NEXT:    movdqa %xmm8, %xmm71478; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm10[0],xmm7[1],xmm10[1],xmm7[2],xmm10[2],xmm7[3],xmm10[3]1479; SSE2-NEXT:    punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm10[4],xmm8[5],xmm10[5],xmm8[6],xmm10[6],xmm8[7],xmm10[7]1480; SSE2-NEXT:    movdqa %xmm8, 240(%rdi)1481; SSE2-NEXT:    movdqa %xmm7, 224(%rdi)1482; SSE2-NEXT:    movdqa %xmm6, 208(%rdi)1483; SSE2-NEXT:    movdqa %xmm15, 192(%rdi)1484; SSE2-NEXT:    movdqa %xmm5, 176(%rdi)1485; SSE2-NEXT:    movdqa %xmm14, 160(%rdi)1486; SSE2-NEXT:    movdqa %xmm4, 144(%rdi)1487; SSE2-NEXT:    movdqa %xmm13, 128(%rdi)1488; SSE2-NEXT:    movdqa %xmm3, 112(%rdi)1489; SSE2-NEXT:    movdqa %xmm12, 96(%rdi)1490; SSE2-NEXT:    movdqa %xmm2, 80(%rdi)1491; SSE2-NEXT:    movdqa %xmm11, 64(%rdi)1492; SSE2-NEXT:    movdqa %xmm1, 48(%rdi)1493; SSE2-NEXT:    movdqa %xmm9, 32(%rdi)1494; SSE2-NEXT:    movdqa %xmm0, 16(%rdi)1495; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1496; SSE2-NEXT:    movaps %xmm0, (%rdi)1497; SSE2-NEXT:    retq1498;1499; SSE41-LABEL: zext_mulhuw_v64i16_lshr:1500; SSE41:       # %bb.0:1501; SSE41-NEXT:    movdqa %xmm0, %xmm81502; SSE41-NEXT:    movq %rdi, %rax1503; SSE41-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm81504; SSE41-NEXT:    pxor %xmm11, %xmm111505; SSE41-NEXT:    movdqa %xmm8, %xmm01506; SSE41-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm11[4],xmm0[5],xmm11[5],xmm0[6],xmm11[6],xmm0[7],xmm11[7]1507; SSE41-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1508; SSE41-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm11509; SSE41-NEXT:    movdqa %xmm1, %xmm91510; SSE41-NEXT:    punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm11[4],xmm9[5],xmm11[5],xmm9[6],xmm11[6],xmm9[7],xmm11[7]1511; SSE41-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm21512; SSE41-NEXT:    movdqa %xmm2, %xmm101513; SSE41-NEXT:    punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm11[4],xmm10[5],xmm11[5],xmm10[6],xmm11[6],xmm10[7],xmm11[7]1514; SSE41-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm31515; SSE41-NEXT:    movdqa %xmm3, %xmm121516; SSE41-NEXT:    punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]1517; SSE41-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm41518; SSE41-NEXT:    movdqa %xmm4, %xmm131519; SSE41-NEXT:    punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm11[4],xmm13[5],xmm11[5],xmm13[6],xmm11[6],xmm13[7],xmm11[7]1520; SSE41-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm51521; SSE41-NEXT:    movdqa %xmm5, %xmm141522; SSE41-NEXT:    punpckhwd {{.*#+}} xmm14 = xmm14[4],xmm11[4],xmm14[5],xmm11[5],xmm14[6],xmm11[6],xmm14[7],xmm11[7]1523; SSE41-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm61524; SSE41-NEXT:    movdqa %xmm6, %xmm151525; SSE41-NEXT:    punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm11[4],xmm15[5],xmm11[5],xmm15[6],xmm11[6],xmm15[7],xmm11[7]1526; SSE41-NEXT:    pmulhuw {{[0-9]+}}(%rsp), %xmm71527; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero1528; SSE41-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm11[4],xmm7[5],xmm11[5],xmm7[6],xmm11[6],xmm7[7],xmm11[7]1529; SSE41-NEXT:    movdqa %xmm7, 240(%rdi)1530; SSE41-NEXT:    movdqa %xmm0, 224(%rdi)1531; SSE41-NEXT:    movdqa %xmm15, 208(%rdi)1532; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero1533; SSE41-NEXT:    movdqa %xmm0, 192(%rdi)1534; SSE41-NEXT:    movdqa %xmm14, 176(%rdi)1535; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero1536; SSE41-NEXT:    movdqa %xmm0, 160(%rdi)1537; SSE41-NEXT:    movdqa %xmm13, 144(%rdi)1538; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero1539; SSE41-NEXT:    movdqa %xmm0, 128(%rdi)1540; SSE41-NEXT:    movdqa %xmm12, 112(%rdi)1541; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero1542; SSE41-NEXT:    movdqa %xmm0, 96(%rdi)1543; SSE41-NEXT:    movdqa %xmm10, 80(%rdi)1544; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero1545; SSE41-NEXT:    movdqa %xmm0, 64(%rdi)1546; SSE41-NEXT:    movdqa %xmm9, 48(%rdi)1547; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero1548; SSE41-NEXT:    movdqa %xmm0, 32(%rdi)1549; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1550; SSE41-NEXT:    movaps %xmm0, 16(%rdi)1551; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm8[0],zero,xmm8[1],zero,xmm8[2],zero,xmm8[3],zero1552; SSE41-NEXT:    movdqa %xmm0, (%rdi)1553; SSE41-NEXT:    retq1554;1555; AVX2-LABEL: zext_mulhuw_v64i16_lshr:1556; AVX2:       # %bb.0:1557; AVX2-NEXT:    movq %rdi, %rax1558; AVX2-NEXT:    vpmulhuw %ymm4, %ymm0, %ymm01559; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1560; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm01561; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1562; AVX2-NEXT:    vpmulhuw %ymm5, %ymm1, %ymm11563; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1564; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm11565; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1566; AVX2-NEXT:    vpmulhuw %ymm6, %ymm2, %ymm21567; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero1568; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm21569; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero1570; AVX2-NEXT:    vpmulhuw %ymm7, %ymm3, %ymm31571; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm7 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero1572; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm31573; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero1574; AVX2-NEXT:    vmovdqa %ymm3, 224(%rdi)1575; AVX2-NEXT:    vmovdqa %ymm7, 192(%rdi)1576; AVX2-NEXT:    vmovdqa %ymm2, 160(%rdi)1577; AVX2-NEXT:    vmovdqa %ymm6, 128(%rdi)1578; AVX2-NEXT:    vmovdqa %ymm1, 96(%rdi)1579; AVX2-NEXT:    vmovdqa %ymm5, 64(%rdi)1580; AVX2-NEXT:    vmovdqa %ymm0, 32(%rdi)1581; AVX2-NEXT:    vmovdqa %ymm4, (%rdi)1582; AVX2-NEXT:    vzeroupper1583; AVX2-NEXT:    retq1584;1585; AVX512F-LABEL: zext_mulhuw_v64i16_lshr:1586; AVX512F:       # %bb.0:1587; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm41588; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero1589; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm21590; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm01591; AVX512F-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm01592; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero1593; AVX512F-NEXT:    vpmulhuw %ymm3, %ymm1, %ymm01594; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero1595; AVX512F-NEXT:    vextracti64x4 $1, %zmm3, %ymm01596; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm11597; AVX512F-NEXT:    vpmulhuw %ymm0, %ymm1, %ymm01598; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero1599; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm01600; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm11601; AVX512F-NEXT:    retq1602;1603; AVX512BW-LABEL: zext_mulhuw_v64i16_lshr:1604; AVX512BW:       # %bb.0:1605; AVX512BW-NEXT:    vpmulhuw %zmm2, %zmm0, %zmm21606; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero1607; AVX512BW-NEXT:    vextracti64x4 $1, %zmm2, %ymm21608; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero1609; AVX512BW-NEXT:    vpmulhuw %zmm3, %zmm1, %zmm11610; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero1611; AVX512BW-NEXT:    vextracti64x4 $1, %zmm1, %ymm11612; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero1613; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm11614; AVX512BW-NEXT:    retq1615  %a1 = zext <64 x i16> %a to <64 x i32>1616  %b1 = zext <64 x i16> %b to <64 x i32>1617  %c = mul <64 x i32> %a1, %b11618  %d = lshr <64 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1619  ret <64 x i32> %d1620}1621 1622define <64 x i32> @mulhsw_v64i16_lshr(<64 x i16> %a, <64 x i16> %b) {1623; SSE2-LABEL: mulhsw_v64i16_lshr:1624; SSE2:       # %bb.0:1625; SSE2-NEXT:    movdqa %xmm7, %xmm81626; SSE2-NEXT:    movq %rdi, %rax1627; SSE2-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm01628; SSE2-NEXT:    pxor %xmm10, %xmm101629; SSE2-NEXT:    movdqa %xmm0, %xmm71630; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm10[0],xmm7[1],xmm10[1],xmm7[2],xmm10[2],xmm7[3],xmm10[3]1631; SSE2-NEXT:    movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1632; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]1633; SSE2-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm11634; SSE2-NEXT:    movdqa %xmm1, %xmm91635; SSE2-NEXT:    punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]1636; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]1637; SSE2-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm21638; SSE2-NEXT:    movdqa %xmm2, %xmm111639; SSE2-NEXT:    punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]1640; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm10[4],xmm2[5],xmm10[5],xmm2[6],xmm10[6],xmm2[7],xmm10[7]1641; SSE2-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm31642; SSE2-NEXT:    movdqa %xmm3, %xmm121643; SSE2-NEXT:    punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]1644; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]1645; SSE2-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm41646; SSE2-NEXT:    movdqa %xmm4, %xmm131647; SSE2-NEXT:    punpcklwd {{.*#+}} xmm13 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3]1648; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm10[4],xmm4[5],xmm10[5],xmm4[6],xmm10[6],xmm4[7],xmm10[7]1649; SSE2-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm51650; SSE2-NEXT:    movdqa %xmm5, %xmm141651; SSE2-NEXT:    punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]1652; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm10[4],xmm5[5],xmm10[5],xmm5[6],xmm10[6],xmm5[7],xmm10[7]1653; SSE2-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm61654; SSE2-NEXT:    movdqa %xmm6, %xmm151655; SSE2-NEXT:    punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm10[0],xmm15[1],xmm10[1],xmm15[2],xmm10[2],xmm15[3],xmm10[3]1656; SSE2-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm10[4],xmm6[5],xmm10[5],xmm6[6],xmm10[6],xmm6[7],xmm10[7]1657; SSE2-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm81658; SSE2-NEXT:    movdqa %xmm8, %xmm71659; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm10[0],xmm7[1],xmm10[1],xmm7[2],xmm10[2],xmm7[3],xmm10[3]1660; SSE2-NEXT:    punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm10[4],xmm8[5],xmm10[5],xmm8[6],xmm10[6],xmm8[7],xmm10[7]1661; SSE2-NEXT:    movdqa %xmm8, 240(%rdi)1662; SSE2-NEXT:    movdqa %xmm7, 224(%rdi)1663; SSE2-NEXT:    movdqa %xmm6, 208(%rdi)1664; SSE2-NEXT:    movdqa %xmm15, 192(%rdi)1665; SSE2-NEXT:    movdqa %xmm5, 176(%rdi)1666; SSE2-NEXT:    movdqa %xmm14, 160(%rdi)1667; SSE2-NEXT:    movdqa %xmm4, 144(%rdi)1668; SSE2-NEXT:    movdqa %xmm13, 128(%rdi)1669; SSE2-NEXT:    movdqa %xmm3, 112(%rdi)1670; SSE2-NEXT:    movdqa %xmm12, 96(%rdi)1671; SSE2-NEXT:    movdqa %xmm2, 80(%rdi)1672; SSE2-NEXT:    movdqa %xmm11, 64(%rdi)1673; SSE2-NEXT:    movdqa %xmm1, 48(%rdi)1674; SSE2-NEXT:    movdqa %xmm9, 32(%rdi)1675; SSE2-NEXT:    movdqa %xmm0, 16(%rdi)1676; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1677; SSE2-NEXT:    movaps %xmm0, (%rdi)1678; SSE2-NEXT:    retq1679;1680; SSE41-LABEL: mulhsw_v64i16_lshr:1681; SSE41:       # %bb.0:1682; SSE41-NEXT:    movdqa %xmm0, %xmm81683; SSE41-NEXT:    movq %rdi, %rax1684; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm81685; SSE41-NEXT:    pxor %xmm11, %xmm111686; SSE41-NEXT:    movdqa %xmm8, %xmm01687; SSE41-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm11[4],xmm0[5],xmm11[5],xmm0[6],xmm11[6],xmm0[7],xmm11[7]1688; SSE41-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1689; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm11690; SSE41-NEXT:    movdqa %xmm1, %xmm91691; SSE41-NEXT:    punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm11[4],xmm9[5],xmm11[5],xmm9[6],xmm11[6],xmm9[7],xmm11[7]1692; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm21693; SSE41-NEXT:    movdqa %xmm2, %xmm101694; SSE41-NEXT:    punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm11[4],xmm10[5],xmm11[5],xmm10[6],xmm11[6],xmm10[7],xmm11[7]1695; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm31696; SSE41-NEXT:    movdqa %xmm3, %xmm121697; SSE41-NEXT:    punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]1698; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm41699; SSE41-NEXT:    movdqa %xmm4, %xmm131700; SSE41-NEXT:    punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm11[4],xmm13[5],xmm11[5],xmm13[6],xmm11[6],xmm13[7],xmm11[7]1701; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm51702; SSE41-NEXT:    movdqa %xmm5, %xmm141703; SSE41-NEXT:    punpckhwd {{.*#+}} xmm14 = xmm14[4],xmm11[4],xmm14[5],xmm11[5],xmm14[6],xmm11[6],xmm14[7],xmm11[7]1704; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm61705; SSE41-NEXT:    movdqa %xmm6, %xmm151706; SSE41-NEXT:    punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm11[4],xmm15[5],xmm11[5],xmm15[6],xmm11[6],xmm15[7],xmm11[7]1707; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm71708; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero1709; SSE41-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm11[4],xmm7[5],xmm11[5],xmm7[6],xmm11[6],xmm7[7],xmm11[7]1710; SSE41-NEXT:    movdqa %xmm7, 240(%rdi)1711; SSE41-NEXT:    movdqa %xmm0, 224(%rdi)1712; SSE41-NEXT:    movdqa %xmm15, 208(%rdi)1713; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero1714; SSE41-NEXT:    movdqa %xmm0, 192(%rdi)1715; SSE41-NEXT:    movdqa %xmm14, 176(%rdi)1716; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero1717; SSE41-NEXT:    movdqa %xmm0, 160(%rdi)1718; SSE41-NEXT:    movdqa %xmm13, 144(%rdi)1719; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero1720; SSE41-NEXT:    movdqa %xmm0, 128(%rdi)1721; SSE41-NEXT:    movdqa %xmm12, 112(%rdi)1722; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero1723; SSE41-NEXT:    movdqa %xmm0, 96(%rdi)1724; SSE41-NEXT:    movdqa %xmm10, 80(%rdi)1725; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero1726; SSE41-NEXT:    movdqa %xmm0, 64(%rdi)1727; SSE41-NEXT:    movdqa %xmm9, 48(%rdi)1728; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero1729; SSE41-NEXT:    movdqa %xmm0, 32(%rdi)1730; SSE41-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1731; SSE41-NEXT:    movaps %xmm0, 16(%rdi)1732; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm8[0],zero,xmm8[1],zero,xmm8[2],zero,xmm8[3],zero1733; SSE41-NEXT:    movdqa %xmm0, (%rdi)1734; SSE41-NEXT:    retq1735;1736; AVX2-LABEL: mulhsw_v64i16_lshr:1737; AVX2:       # %bb.0:1738; AVX2-NEXT:    movq %rdi, %rax1739; AVX2-NEXT:    vpmulhw %ymm4, %ymm0, %ymm01740; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1741; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm01742; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1743; AVX2-NEXT:    vpmulhw %ymm5, %ymm1, %ymm11744; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1745; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm11746; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1747; AVX2-NEXT:    vpmulhw %ymm6, %ymm2, %ymm21748; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero1749; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm21750; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero1751; AVX2-NEXT:    vpmulhw %ymm7, %ymm3, %ymm31752; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm7 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero1753; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm31754; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero1755; AVX2-NEXT:    vmovdqa %ymm3, 224(%rdi)1756; AVX2-NEXT:    vmovdqa %ymm7, 192(%rdi)1757; AVX2-NEXT:    vmovdqa %ymm2, 160(%rdi)1758; AVX2-NEXT:    vmovdqa %ymm6, 128(%rdi)1759; AVX2-NEXT:    vmovdqa %ymm1, 96(%rdi)1760; AVX2-NEXT:    vmovdqa %ymm5, 64(%rdi)1761; AVX2-NEXT:    vmovdqa %ymm0, 32(%rdi)1762; AVX2-NEXT:    vmovdqa %ymm4, (%rdi)1763; AVX2-NEXT:    vzeroupper1764; AVX2-NEXT:    retq1765;1766; AVX512F-LABEL: mulhsw_v64i16_lshr:1767; AVX512F:       # %bb.0:1768; AVX512F-NEXT:    vpmulhw %ymm2, %ymm0, %ymm41769; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero1770; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm21771; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm01772; AVX512F-NEXT:    vpmulhw %ymm2, %ymm0, %ymm01773; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm5 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero1774; AVX512F-NEXT:    vpmulhw %ymm3, %ymm1, %ymm01775; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero1776; AVX512F-NEXT:    vextracti64x4 $1, %zmm3, %ymm01777; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm11778; AVX512F-NEXT:    vpmulhw %ymm0, %ymm1, %ymm01779; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero1780; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm01781; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm11782; AVX512F-NEXT:    retq1783;1784; AVX512BW-LABEL: mulhsw_v64i16_lshr:1785; AVX512BW:       # %bb.0:1786; AVX512BW-NEXT:    vpmulhw %zmm2, %zmm0, %zmm21787; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero1788; AVX512BW-NEXT:    vextracti64x4 $1, %zmm2, %ymm21789; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero1790; AVX512BW-NEXT:    vpmulhw %zmm3, %zmm1, %zmm11791; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero1792; AVX512BW-NEXT:    vextracti64x4 $1, %zmm1, %ymm11793; AVX512BW-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero1794; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm11795; AVX512BW-NEXT:    retq1796  %a1 = sext <64 x i16> %a to <64 x i32>1797  %b1 = sext <64 x i16> %b to <64 x i32>1798  %c = mul <64 x i32> %a1, %b11799  %d = lshr <64 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1800  ret <64 x i32> %d1801}1802 1803define <64 x i32> @mulhsw_v64i16_ashr(<64 x i16> %a, <64 x i16> %b) {1804; SSE2-LABEL: mulhsw_v64i16_ashr:1805; SSE2:       # %bb.0:1806; SSE2-NEXT:    movq %rdi, %rax1807; SSE2-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm01808; SSE2-NEXT:    punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]1809; SSE2-NEXT:    psrad $16, %xmm81810; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]1811; SSE2-NEXT:    psrad $16, %xmm01812; SSE2-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm11813; SSE2-NEXT:    punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm1[0],xmm9[1],xmm1[1],xmm9[2],xmm1[2],xmm9[3],xmm1[3]1814; SSE2-NEXT:    psrad $16, %xmm91815; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]1816; SSE2-NEXT:    psrad $16, %xmm11817; SSE2-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm21818; SSE2-NEXT:    punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm2[0],xmm10[1],xmm2[1],xmm10[2],xmm2[2],xmm10[3],xmm2[3]1819; SSE2-NEXT:    psrad $16, %xmm101820; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]1821; SSE2-NEXT:    psrad $16, %xmm21822; SSE2-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm31823; SSE2-NEXT:    punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm3[0],xmm11[1],xmm3[1],xmm11[2],xmm3[2],xmm11[3],xmm3[3]1824; SSE2-NEXT:    psrad $16, %xmm111825; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]1826; SSE2-NEXT:    psrad $16, %xmm31827; SSE2-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm41828; SSE2-NEXT:    punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm4[0],xmm12[1],xmm4[1],xmm12[2],xmm4[2],xmm12[3],xmm4[3]1829; SSE2-NEXT:    psrad $16, %xmm121830; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]1831; SSE2-NEXT:    psrad $16, %xmm41832; SSE2-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm51833; SSE2-NEXT:    punpcklwd {{.*#+}} xmm13 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]1834; SSE2-NEXT:    psrad $16, %xmm131835; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]1836; SSE2-NEXT:    psrad $16, %xmm51837; SSE2-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm61838; SSE2-NEXT:    punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]1839; SSE2-NEXT:    psrad $16, %xmm141840; SSE2-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4,4,5,5,6,6,7,7]1841; SSE2-NEXT:    psrad $16, %xmm61842; SSE2-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm71843; SSE2-NEXT:    punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm7[0],xmm15[1],xmm7[1],xmm15[2],xmm7[2],xmm15[3],xmm7[3]1844; SSE2-NEXT:    psrad $16, %xmm151845; SSE2-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4,4,5,5,6,6,7,7]1846; SSE2-NEXT:    psrad $16, %xmm71847; SSE2-NEXT:    movdqa %xmm7, 240(%rdi)1848; SSE2-NEXT:    movdqa %xmm15, 224(%rdi)1849; SSE2-NEXT:    movdqa %xmm6, 208(%rdi)1850; SSE2-NEXT:    movdqa %xmm14, 192(%rdi)1851; SSE2-NEXT:    movdqa %xmm5, 176(%rdi)1852; SSE2-NEXT:    movdqa %xmm13, 160(%rdi)1853; SSE2-NEXT:    movdqa %xmm4, 144(%rdi)1854; SSE2-NEXT:    movdqa %xmm12, 128(%rdi)1855; SSE2-NEXT:    movdqa %xmm3, 112(%rdi)1856; SSE2-NEXT:    movdqa %xmm11, 96(%rdi)1857; SSE2-NEXT:    movdqa %xmm2, 80(%rdi)1858; SSE2-NEXT:    movdqa %xmm10, 64(%rdi)1859; SSE2-NEXT:    movdqa %xmm1, 48(%rdi)1860; SSE2-NEXT:    movdqa %xmm9, 32(%rdi)1861; SSE2-NEXT:    movdqa %xmm0, 16(%rdi)1862; SSE2-NEXT:    movdqa %xmm8, (%rdi)1863; SSE2-NEXT:    retq1864;1865; SSE41-LABEL: mulhsw_v64i16_ashr:1866; SSE41:       # %bb.0:1867; SSE41-NEXT:    movq %rdi, %rax1868; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm01869; SSE41-NEXT:    pmovsxwd %xmm0, %xmm81870; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]1871; SSE41-NEXT:    pmovsxwd %xmm0, %xmm01872; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm11873; SSE41-NEXT:    pmovsxwd %xmm1, %xmm91874; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]1875; SSE41-NEXT:    pmovsxwd %xmm1, %xmm11876; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm21877; SSE41-NEXT:    pmovsxwd %xmm2, %xmm101878; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]1879; SSE41-NEXT:    pmovsxwd %xmm2, %xmm21880; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm31881; SSE41-NEXT:    pmovsxwd %xmm3, %xmm111882; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]1883; SSE41-NEXT:    pmovsxwd %xmm3, %xmm31884; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm41885; SSE41-NEXT:    pmovsxwd %xmm4, %xmm121886; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]1887; SSE41-NEXT:    pmovsxwd %xmm4, %xmm41888; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm51889; SSE41-NEXT:    pmovsxwd %xmm5, %xmm131890; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]1891; SSE41-NEXT:    pmovsxwd %xmm5, %xmm51892; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm61893; SSE41-NEXT:    pmovsxwd %xmm6, %xmm141894; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]1895; SSE41-NEXT:    pmovsxwd %xmm6, %xmm61896; SSE41-NEXT:    pmulhw {{[0-9]+}}(%rsp), %xmm71897; SSE41-NEXT:    pmovsxwd %xmm7, %xmm151898; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[2,3,2,3]1899; SSE41-NEXT:    pmovsxwd %xmm7, %xmm71900; SSE41-NEXT:    movdqa %xmm7, 240(%rdi)1901; SSE41-NEXT:    movdqa %xmm15, 224(%rdi)1902; SSE41-NEXT:    movdqa %xmm6, 208(%rdi)1903; SSE41-NEXT:    movdqa %xmm14, 192(%rdi)1904; SSE41-NEXT:    movdqa %xmm5, 176(%rdi)1905; SSE41-NEXT:    movdqa %xmm13, 160(%rdi)1906; SSE41-NEXT:    movdqa %xmm4, 144(%rdi)1907; SSE41-NEXT:    movdqa %xmm12, 128(%rdi)1908; SSE41-NEXT:    movdqa %xmm3, 112(%rdi)1909; SSE41-NEXT:    movdqa %xmm11, 96(%rdi)1910; SSE41-NEXT:    movdqa %xmm2, 80(%rdi)1911; SSE41-NEXT:    movdqa %xmm10, 64(%rdi)1912; SSE41-NEXT:    movdqa %xmm1, 48(%rdi)1913; SSE41-NEXT:    movdqa %xmm9, 32(%rdi)1914; SSE41-NEXT:    movdqa %xmm0, 16(%rdi)1915; SSE41-NEXT:    movdqa %xmm8, (%rdi)1916; SSE41-NEXT:    retq1917;1918; AVX2-LABEL: mulhsw_v64i16_ashr:1919; AVX2:       # %bb.0:1920; AVX2-NEXT:    movq %rdi, %rax1921; AVX2-NEXT:    vpmulhw %ymm4, %ymm0, %ymm01922; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm41923; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm01924; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm01925; AVX2-NEXT:    vpmulhw %ymm5, %ymm1, %ymm11926; AVX2-NEXT:    vpmovsxwd %xmm1, %ymm51927; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm11928; AVX2-NEXT:    vpmovsxwd %xmm1, %ymm11929; AVX2-NEXT:    vpmulhw %ymm6, %ymm2, %ymm21930; AVX2-NEXT:    vpmovsxwd %xmm2, %ymm61931; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm21932; AVX2-NEXT:    vpmovsxwd %xmm2, %ymm21933; AVX2-NEXT:    vpmulhw %ymm7, %ymm3, %ymm31934; AVX2-NEXT:    vpmovsxwd %xmm3, %ymm71935; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm31936; AVX2-NEXT:    vpmovsxwd %xmm3, %ymm31937; AVX2-NEXT:    vmovdqa %ymm3, 224(%rdi)1938; AVX2-NEXT:    vmovdqa %ymm7, 192(%rdi)1939; AVX2-NEXT:    vmovdqa %ymm2, 160(%rdi)1940; AVX2-NEXT:    vmovdqa %ymm6, 128(%rdi)1941; AVX2-NEXT:    vmovdqa %ymm1, 96(%rdi)1942; AVX2-NEXT:    vmovdqa %ymm5, 64(%rdi)1943; AVX2-NEXT:    vmovdqa %ymm0, 32(%rdi)1944; AVX2-NEXT:    vmovdqa %ymm4, (%rdi)1945; AVX2-NEXT:    vzeroupper1946; AVX2-NEXT:    retq1947;1948; AVX512F-LABEL: mulhsw_v64i16_ashr:1949; AVX512F:       # %bb.0:1950; AVX512F-NEXT:    vpmulhw %ymm2, %ymm0, %ymm41951; AVX512F-NEXT:    vpmovsxwd %ymm4, %zmm41952; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm21953; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm01954; AVX512F-NEXT:    vpmulhw %ymm2, %ymm0, %ymm01955; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm51956; AVX512F-NEXT:    vpmulhw %ymm3, %ymm1, %ymm01957; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm21958; AVX512F-NEXT:    vextracti64x4 $1, %zmm3, %ymm01959; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm11960; AVX512F-NEXT:    vpmulhw %ymm0, %ymm1, %ymm01961; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm31962; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm01963; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm11964; AVX512F-NEXT:    retq1965;1966; AVX512BW-LABEL: mulhsw_v64i16_ashr:1967; AVX512BW:       # %bb.0:1968; AVX512BW-NEXT:    vpmulhw %zmm2, %zmm0, %zmm21969; AVX512BW-NEXT:    vpmovsxwd %ymm2, %zmm01970; AVX512BW-NEXT:    vextracti64x4 $1, %zmm2, %ymm21971; AVX512BW-NEXT:    vpmovsxwd %ymm2, %zmm41972; AVX512BW-NEXT:    vpmulhw %zmm3, %zmm1, %zmm11973; AVX512BW-NEXT:    vpmovsxwd %ymm1, %zmm21974; AVX512BW-NEXT:    vextracti64x4 $1, %zmm1, %ymm11975; AVX512BW-NEXT:    vpmovsxwd %ymm1, %zmm31976; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm11977; AVX512BW-NEXT:    retq1978  %a1 = sext <64 x i16> %a to <64 x i32>1979  %b1 = sext <64 x i16> %b to <64 x i32>1980  %c = mul <64 x i32> %a1, %b11981  %d = ashr <64 x i32> %c, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1982  ret <64 x i32> %d1983}1984 1985define <8 x i64> @zext_mulhuw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {1986; SSE2-LABEL: zext_mulhuw_v8i16_lshr_i64:1987; SSE2:       # %bb.0:1988; SSE2-NEXT:    movdqa %xmm0, %xmm31989; SSE2-NEXT:    pmulhuw %xmm1, %xmm31990; SSE2-NEXT:    pxor %xmm4, %xmm41991; SSE2-NEXT:    movdqa %xmm3, %xmm11992; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]1993; SSE2-NEXT:    movdqa %xmm1, %xmm01994; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]1995; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm4[2],xmm1[3],xmm4[3]1996; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]1997; SSE2-NEXT:    movdqa %xmm3, %xmm21998; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]1999; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]2000; SSE2-NEXT:    retq2001;2002; SSE41-LABEL: zext_mulhuw_v8i16_lshr_i64:2003; SSE41:       # %bb.0:2004; SSE41-NEXT:    pmulhuw %xmm1, %xmm02005; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero2006; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]2007; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero2008; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]2009; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero2010; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]2011; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero2012; SSE41-NEXT:    movdqa %xmm4, %xmm02013; SSE41-NEXT:    retq2014;2015; AVX2-LABEL: zext_mulhuw_v8i16_lshr_i64:2016; AVX2:       # %bb.0:2017; AVX2-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm12018; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero2019; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]2020; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero2021; AVX2-NEXT:    retq2022;2023; AVX512-LABEL: zext_mulhuw_v8i16_lshr_i64:2024; AVX512:       # %bb.0:2025; AVX512-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm02026; AVX512-NEXT:    vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero2027; AVX512-NEXT:    retq2028  %a1 = zext <8 x i16> %a to <8 x i64>2029  %b1 = zext <8 x i16> %b to <8 x i64>2030  %c = mul <8 x i64> %a1, %b12031  %d = lshr <8 x i64> %c, <i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16>2032  ret <8 x i64> %d2033}2034 2035define <8 x i64> @sext_mulhsw_v8i16_lshr_i64(<8 x i16> %a, <8 x i16> %b) {2036; SSE2-LABEL: sext_mulhsw_v8i16_lshr_i64:2037; SSE2:       # %bb.0:2038; SSE2-NEXT:    movdqa %xmm0, %xmm32039; SSE2-NEXT:    pmulhw %xmm1, %xmm32040; SSE2-NEXT:    pxor %xmm4, %xmm42041; SSE2-NEXT:    movdqa %xmm3, %xmm12042; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]2043; SSE2-NEXT:    movdqa %xmm1, %xmm02044; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]2045; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm4[2],xmm1[3],xmm4[3]2046; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]2047; SSE2-NEXT:    movdqa %xmm3, %xmm22048; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]2049; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]2050; SSE2-NEXT:    retq2051;2052; SSE41-LABEL: sext_mulhsw_v8i16_lshr_i64:2053; SSE41:       # %bb.0:2054; SSE41-NEXT:    pmulhw %xmm1, %xmm02055; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero2056; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]2057; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero2058; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]2059; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero2060; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]2061; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero2062; SSE41-NEXT:    movdqa %xmm4, %xmm02063; SSE41-NEXT:    retq2064;2065; AVX2-LABEL: sext_mulhsw_v8i16_lshr_i64:2066; AVX2:       # %bb.0:2067; AVX2-NEXT:    vpmulhw %xmm1, %xmm0, %xmm12068; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero2069; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]2070; AVX2-NEXT:    vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero2071; AVX2-NEXT:    retq2072;2073; AVX512-LABEL: sext_mulhsw_v8i16_lshr_i64:2074; AVX512:       # %bb.0:2075; AVX512-NEXT:    vpmulhw %xmm1, %xmm0, %xmm02076; AVX512-NEXT:    vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero2077; AVX512-NEXT:    retq2078  %a1 = sext <8 x i16> %a to <8 x i64>2079  %b1 = sext <8 x i16> %b to <8 x i64>2080  %c = mul <8 x i64> %a1, %b12081  %d = lshr <8 x i64> %c, <i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16>2082  ret <8 x i64> %d2083}2084 2085define <8 x i64> @sext_mulhsw_v8i16_ashr_i64(<8 x i16> %a, <8 x i16> %b) {2086; SSE2-LABEL: sext_mulhsw_v8i16_ashr_i64:2087; SSE2:       # %bb.0:2088; SSE2-NEXT:    pmulhw %xmm1, %xmm02089; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]2090; SSE2-NEXT:    psrad $16, %xmm12091; SSE2-NEXT:    pxor %xmm5, %xmm52092; SSE2-NEXT:    pxor %xmm2, %xmm22093; SSE2-NEXT:    pcmpgtd %xmm1, %xmm22094; SSE2-NEXT:    movdqa %xmm1, %xmm42095; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]2096; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]2097; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]2098; SSE2-NEXT:    psrad $16, %xmm32099; SSE2-NEXT:    pcmpgtd %xmm3, %xmm52100; SSE2-NEXT:    movdqa %xmm3, %xmm22101; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]2102; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]2103; SSE2-NEXT:    movdqa %xmm4, %xmm02104; SSE2-NEXT:    retq2105;2106; SSE41-LABEL: sext_mulhsw_v8i16_ashr_i64:2107; SSE41:       # %bb.0:2108; SSE41-NEXT:    pmulhw %xmm1, %xmm02109; SSE41-NEXT:    pmovsxwq %xmm0, %xmm42110; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]2111; SSE41-NEXT:    pmovsxwq %xmm1, %xmm12112; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]2113; SSE41-NEXT:    pmovsxwq %xmm2, %xmm22114; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]2115; SSE41-NEXT:    pmovsxwq %xmm0, %xmm32116; SSE41-NEXT:    movdqa %xmm4, %xmm02117; SSE41-NEXT:    retq2118;2119; AVX2-LABEL: sext_mulhsw_v8i16_ashr_i64:2120; AVX2:       # %bb.0:2121; AVX2-NEXT:    vpmulhw %xmm1, %xmm0, %xmm12122; AVX2-NEXT:    vpmovsxwq %xmm1, %ymm02123; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]2124; AVX2-NEXT:    vpmovsxwq %xmm1, %ymm12125; AVX2-NEXT:    retq2126;2127; AVX512-LABEL: sext_mulhsw_v8i16_ashr_i64:2128; AVX512:       # %bb.0:2129; AVX512-NEXT:    vpmulhw %xmm1, %xmm0, %xmm02130; AVX512-NEXT:    vpmovsxwq %xmm0, %zmm02131; AVX512-NEXT:    retq2132  %a1 = sext <8 x i16> %a to <8 x i64>2133  %b1 = sext <8 x i16> %b to <8 x i64>2134  %c = mul <8 x i64> %a1, %b12135  %d = ashr <8 x i64> %c, <i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16>2136  ret <8 x i64> %d2137}2138 2139define <8 x i16> @sse2_pmulh_w_const(<8 x i16> %a0, <8 x i16> %a1) {2140; SSE-LABEL: sse2_pmulh_w_const:2141; SSE:       # %bb.0:2142; SSE-NEXT:    movaps {{.*#+}} xmm0 = [0,65535,65535,65535,65535,65535,65535,0]2143; SSE-NEXT:    retq2144;2145; AVX-LABEL: sse2_pmulh_w_const:2146; AVX:       # %bb.0:2147; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [0,65535,65535,65535,65535,65535,65535,0]2148; AVX-NEXT:    retq2149  %res = call <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16> <i16 -1, i16 -2, i16 -3, i16 -4, i16 -5, i16 -6, i16 -7, i16 0>, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)2150  ret <8 x i16> %res2151}2152declare <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16>, <8 x i16>)2153 2154define <8 x i16> @sse2_pmulhu_w_const(<8 x i16> %a0, <8 x i16> %a1) {2155; SSE-LABEL: sse2_pmulhu_w_const:2156; SSE:       # %bb.0:2157; SSE-NEXT:    movaps {{.*#+}} xmm0 = [0,0,1,2,3,4,5,0]2158; SSE-NEXT:    retq2159;2160; AVX-LABEL: sse2_pmulhu_w_const:2161; AVX:       # %bb.0:2162; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [0,0,1,2,3,4,5,0]2163; AVX-NEXT:    retq2164  %res = call <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16> <i16 -1, i16 -2, i16 -3, i16 -4, i16 -5, i16 -6, i16 -7, i16 0>, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)2165  ret <8 x i16> %res2166}2167declare <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16>, <8 x i16>)2168 2169define <8 x i16> @zext_mul_and_shift17(<8 x i16> %a, <8 x i16> %b) {2170; SSE-LABEL: zext_mul_and_shift17:2171; SSE:       # %bb.0:2172; SSE-NEXT:    pmulhuw %xmm1, %xmm02173; SSE-NEXT:    psrlw $1, %xmm02174; SSE-NEXT:    retq2175;2176; AVX-LABEL: zext_mul_and_shift17:2177; AVX:       # %bb.0:2178; AVX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm02179; AVX-NEXT:    vpsrlw $1, %xmm0, %xmm02180; AVX-NEXT:    retq2181  %a.ext = zext <8 x i16> %a to <8 x i32>2182  %b.ext = zext <8 x i16> %b to <8 x i32>2183  %mul = mul <8 x i32> %a.ext, %b.ext2184  %shift = lshr <8 x i32> %mul, splat(i32 17)2185  %trunc = trunc <8 x i32> %shift to <8 x i16>2186  ret <8 x i16> %trunc2187}2188 2189define <8 x i16> @zext_mul_and_shift24(<8 x i16> %a, <8 x i16> %b) {2190; SSE-LABEL: zext_mul_and_shift24:2191; SSE:       # %bb.0:2192; SSE-NEXT:    pmulhuw %xmm1, %xmm02193; SSE-NEXT:    psrlw $8, %xmm02194; SSE-NEXT:    retq2195;2196; AVX-LABEL: zext_mul_and_shift24:2197; AVX:       # %bb.0:2198; AVX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm02199; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm02200; AVX-NEXT:    retq2201  %a.ext = zext <8 x i16> %a to <8 x i32>2202  %b.ext = zext <8 x i16> %b to <8 x i32>2203  %mul = mul <8 x i32> %a.ext, %b.ext2204  %shift = lshr <8 x i32> %mul, splat(i32 24)2205  %trunc = trunc <8 x i32> %shift to <8 x i16>2206  ret <8 x i16> %trunc2207}2208 2209define <8 x i16> @zext_mul_and_shift31(<8 x i16> %a, <8 x i16> %b) {2210; SSE-LABEL: zext_mul_and_shift31:2211; SSE:       # %bb.0:2212; SSE-NEXT:    pmulhuw %xmm1, %xmm02213; SSE-NEXT:    psrlw $15, %xmm02214; SSE-NEXT:    retq2215;2216; AVX-LABEL: zext_mul_and_shift31:2217; AVX:       # %bb.0:2218; AVX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm02219; AVX-NEXT:    vpsrlw $15, %xmm0, %xmm02220; AVX-NEXT:    retq2221  %a.ext = zext <8 x i16> %a to <8 x i32>2222  %b.ext = zext <8 x i16> %b to <8 x i32>2223  %mul = mul <8 x i32> %a.ext, %b.ext2224  %shift = lshr <8 x i32> %mul, splat(i32 31)2225  %trunc = trunc <8 x i32> %shift to <8 x i16>2226  ret <8 x i16> %trunc2227}2228 2229define <8 x i16> @sext_mul_and_shift17(<8 x i16> %a, <8 x i16> %b) {2230; SSE-LABEL: sext_mul_and_shift17:2231; SSE:       # %bb.0:2232; SSE-NEXT:    pmulhw %xmm1, %xmm02233; SSE-NEXT:    psrlw $1, %xmm02234; SSE-NEXT:    retq2235;2236; AVX-LABEL: sext_mul_and_shift17:2237; AVX:       # %bb.0:2238; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm02239; AVX-NEXT:    vpsrlw $1, %xmm0, %xmm02240; AVX-NEXT:    retq2241  %a.ext = sext <8 x i16> %a to <8 x i32>2242  %b.ext = sext <8 x i16> %b to <8 x i32>2243  %mul = mul <8 x i32> %a.ext, %b.ext2244  %shift = lshr <8 x i32> %mul, splat(i32 17)2245  %trunc = trunc <8 x i32> %shift to <8 x i16>2246  ret <8 x i16> %trunc2247}2248 2249define <8 x i16> @sext_mul_and_shift24(<8 x i16> %a, <8 x i16> %b) {2250; SSE-LABEL: sext_mul_and_shift24:2251; SSE:       # %bb.0:2252; SSE-NEXT:    pmulhw %xmm1, %xmm02253; SSE-NEXT:    psrlw $8, %xmm02254; SSE-NEXT:    retq2255;2256; AVX-LABEL: sext_mul_and_shift24:2257; AVX:       # %bb.0:2258; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm02259; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm02260; AVX-NEXT:    retq2261  %a.ext = sext <8 x i16> %a to <8 x i32>2262  %b.ext = sext <8 x i16> %b to <8 x i32>2263  %mul = mul <8 x i32> %a.ext, %b.ext2264  %shift = lshr <8 x i32> %mul, splat(i32 24)2265  %trunc = trunc <8 x i32> %shift to <8 x i16>2266  ret <8 x i16> %trunc2267}2268 2269define <8 x i16> @sext_mul_and_shift31(<8 x i16> %a, <8 x i16> %b) {2270; SSE-LABEL: sext_mul_and_shift31:2271; SSE:       # %bb.0:2272; SSE-NEXT:    pmulhw %xmm1, %xmm02273; SSE-NEXT:    psrlw $15, %xmm02274; SSE-NEXT:    retq2275;2276; AVX-LABEL: sext_mul_and_shift31:2277; AVX:       # %bb.0:2278; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm02279; AVX-NEXT:    vpsrlw $15, %xmm0, %xmm02280; AVX-NEXT:    retq2281  %a.ext = sext <8 x i16> %a to <8 x i32>2282  %b.ext = sext <8 x i16> %b to <8 x i32>2283  %mul = mul <8 x i32> %a.ext, %b.ext2284  %shift = lshr <8 x i32> %mul, splat(i32 31)2285  %trunc = trunc <8 x i32> %shift to <8 x i16>2286  ret <8 x i16> %trunc2287}2288 2289define <4 x i16> @sext_mulhw_v4i16_shift17(<4 x i16> %a, <4 x i16> %b) {2290; SSE-LABEL: sext_mulhw_v4i16_shift17:2291; SSE:       # %bb.0:2292; SSE-NEXT:    pmulhw %xmm1, %xmm02293; SSE-NEXT:    psrlw $1, %xmm02294; SSE-NEXT:    retq2295;2296; AVX-LABEL: sext_mulhw_v4i16_shift17:2297; AVX:       # %bb.0:2298; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm02299; AVX-NEXT:    vpsrlw $1, %xmm0, %xmm02300; AVX-NEXT:    retq2301  %a1 = sext <4 x i16> %a to <4 x i32>2302  %b1 = sext <4 x i16> %b to <4 x i32>2303  %c = mul <4 x i32> %a1, %b12304  %d = lshr <4 x i32> %c, splat (i32 17)2305  %e = trunc <4 x i32> %d to <4 x i16>2306  ret <4 x i16> %e2307}2308 2309define <4 x i16> @sext_mulhw_v4i16_shift24(<4 x i16> %a, <4 x i16> %b) {2310; SSE-LABEL: sext_mulhw_v4i16_shift24:2311; SSE:       # %bb.0:2312; SSE-NEXT:    pmulhw %xmm1, %xmm02313; SSE-NEXT:    psrlw $8, %xmm02314; SSE-NEXT:    retq2315;2316; AVX-LABEL: sext_mulhw_v4i16_shift24:2317; AVX:       # %bb.0:2318; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm02319; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm02320; AVX-NEXT:    retq2321  %a1 = sext <4 x i16> %a to <4 x i32>2322  %b1 = sext <4 x i16> %b to <4 x i32>2323  %c = mul <4 x i32> %a1, %b12324  %d = lshr <4 x i32> %c, splat (i32 24)2325  %e = trunc <4 x i32> %d to <4 x i16>2326  ret <4 x i16> %e2327}2328 2329define <4 x i16> @sext_mulhw_v4i16_shift31(<4 x i16> %a, <4 x i16> %b) {2330; SSE-LABEL: sext_mulhw_v4i16_shift31:2331; SSE:       # %bb.0:2332; SSE-NEXT:    pmulhw %xmm1, %xmm02333; SSE-NEXT:    psrlw $15, %xmm02334; SSE-NEXT:    retq2335;2336; AVX-LABEL: sext_mulhw_v4i16_shift31:2337; AVX:       # %bb.0:2338; AVX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm02339; AVX-NEXT:    vpsrlw $15, %xmm0, %xmm02340; AVX-NEXT:    retq2341  %a1 = sext <4 x i16> %a to <4 x i32>2342  %b1 = sext <4 x i16> %b to <4 x i32>2343  %c = mul <4 x i32> %a1, %b12344  %d = lshr <4 x i32> %c, splat (i32 31)2345  %e = trunc <4 x i32> %d to <4 x i16>2346  ret <4 x i16> %e2347}2348 2349define <4 x i16> @and_mulhuw_v4i16_shift17(<4 x i64> %a, <4 x i64> %b) {2350; SSE2-LABEL: and_mulhuw_v4i16_shift17:2351; SSE2:       # %bb.0:2352; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]2353; SSE2-NEXT:    pslld $16, %xmm22354; SSE2-NEXT:    psrad $16, %xmm22355; SSE2-NEXT:    xorps %xmm3, %xmm32356; SSE2-NEXT:    packssdw %xmm3, %xmm22357; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]2358; SSE2-NEXT:    pslld $16, %xmm02359; SSE2-NEXT:    psrad $16, %xmm02360; SSE2-NEXT:    packssdw %xmm3, %xmm02361; SSE2-NEXT:    pmulhuw %xmm2, %xmm02362; SSE2-NEXT:    psrlw $1, %xmm02363; SSE2-NEXT:    retq2364;2365; SSE41-LABEL: and_mulhuw_v4i16_shift17:2366; SSE41:       # %bb.0:2367; SSE41-NEXT:    pxor %xmm4, %xmm42368; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1,2,3],xmm3[4],xmm4[5,6,7]2369; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1,2,3],xmm2[4],xmm4[5,6,7]2370; SSE41-NEXT:    packusdw %xmm3, %xmm22371; SSE41-NEXT:    packusdw %xmm4, %xmm22372; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm4[1,2,3],xmm1[4],xmm4[5,6,7]2373; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm4[1,2,3],xmm0[4],xmm4[5,6,7]2374; SSE41-NEXT:    packusdw %xmm1, %xmm02375; SSE41-NEXT:    packusdw %xmm4, %xmm02376; SSE41-NEXT:    pmulhuw %xmm2, %xmm02377; SSE41-NEXT:    psrlw $1, %xmm02378; SSE41-NEXT:    retq2379;2380; AVX2-LABEL: and_mulhuw_v4i16_shift17:2381; AVX2:       # %bb.0:2382; AVX2-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm02383; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm12384; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15]2385; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm12386; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm02387; AVX2-NEXT:    vpackusdw %xmm0, %xmm0, %xmm02388; AVX2-NEXT:    vpsrlw $1, %xmm0, %xmm02389; AVX2-NEXT:    vzeroupper2390; AVX2-NEXT:    retq2391;2392; AVX512-LABEL: and_mulhuw_v4i16_shift17:2393; AVX512:       # %bb.0:2394; AVX512-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm02395; AVX512-NEXT:    vpmovqw %zmm0, %xmm02396; AVX512-NEXT:    vpsrlw $1, %xmm0, %xmm02397; AVX512-NEXT:    vzeroupper2398; AVX512-NEXT:    retq2399  %a1 = and <4 x i64> %a, <i64 65535, i64 65535, i64 65535, i64 65535>2400  %b1 = and <4 x i64> %b, <i64 65535, i64 65535, i64 65535, i64 65535>2401  %c = mul <4 x i64> %a1, %b12402  %d = lshr <4 x i64> %c, splat (i64 17)2403  %e = trunc <4 x i64> %d to <4 x i16>2404  ret <4 x i16> %e2405}2406 2407define <4 x i16> @and_mulhuw_v4i16_shift24(<4 x i64> %a, <4 x i64> %b) {2408; SSE2-LABEL: and_mulhuw_v4i16_shift24:2409; SSE2:       # %bb.0:2410; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]2411; SSE2-NEXT:    pslld $16, %xmm22412; SSE2-NEXT:    psrad $16, %xmm22413; SSE2-NEXT:    xorps %xmm3, %xmm32414; SSE2-NEXT:    packssdw %xmm3, %xmm22415; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]2416; SSE2-NEXT:    pslld $16, %xmm02417; SSE2-NEXT:    psrad $16, %xmm02418; SSE2-NEXT:    packssdw %xmm3, %xmm02419; SSE2-NEXT:    pmulhuw %xmm2, %xmm02420; SSE2-NEXT:    psrlw $8, %xmm02421; SSE2-NEXT:    retq2422;2423; SSE41-LABEL: and_mulhuw_v4i16_shift24:2424; SSE41:       # %bb.0:2425; SSE41-NEXT:    pxor %xmm4, %xmm42426; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1,2,3],xmm3[4],xmm4[5,6,7]2427; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1,2,3],xmm2[4],xmm4[5,6,7]2428; SSE41-NEXT:    packusdw %xmm3, %xmm22429; SSE41-NEXT:    packusdw %xmm4, %xmm22430; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm4[1,2,3],xmm1[4],xmm4[5,6,7]2431; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm4[1,2,3],xmm0[4],xmm4[5,6,7]2432; SSE41-NEXT:    packusdw %xmm1, %xmm02433; SSE41-NEXT:    packusdw %xmm4, %xmm02434; SSE41-NEXT:    pmulhuw %xmm2, %xmm02435; SSE41-NEXT:    psrlw $8, %xmm02436; SSE41-NEXT:    retq2437;2438; AVX2-LABEL: and_mulhuw_v4i16_shift24:2439; AVX2:       # %bb.0:2440; AVX2-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm02441; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm12442; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15]2443; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm12444; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm02445; AVX2-NEXT:    vpackusdw %xmm0, %xmm0, %xmm02446; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm02447; AVX2-NEXT:    vzeroupper2448; AVX2-NEXT:    retq2449;2450; AVX512-LABEL: and_mulhuw_v4i16_shift24:2451; AVX512:       # %bb.0:2452; AVX512-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm02453; AVX512-NEXT:    vpmovqw %zmm0, %xmm02454; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm02455; AVX512-NEXT:    vzeroupper2456; AVX512-NEXT:    retq2457  %a1 = and <4 x i64> %a, <i64 65535, i64 65535, i64 65535, i64 65535>2458  %b1 = and <4 x i64> %b, <i64 65535, i64 65535, i64 65535, i64 65535>2459  %c = mul <4 x i64> %a1, %b12460  %d = lshr <4 x i64> %c, splat (i64 24)2461  %e = trunc <4 x i64> %d to <4 x i16>2462  ret <4 x i16> %e2463}2464 2465define <4 x i16> @and_mulhuw_v4i16_shift31(<4 x i64> %a, <4 x i64> %b) {2466; SSE2-LABEL: and_mulhuw_v4i16_shift31:2467; SSE2:       # %bb.0:2468; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]2469; SSE2-NEXT:    pslld $16, %xmm22470; SSE2-NEXT:    psrad $16, %xmm22471; SSE2-NEXT:    xorps %xmm3, %xmm32472; SSE2-NEXT:    packssdw %xmm3, %xmm22473; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]2474; SSE2-NEXT:    pslld $16, %xmm02475; SSE2-NEXT:    psrad $16, %xmm02476; SSE2-NEXT:    packssdw %xmm3, %xmm02477; SSE2-NEXT:    pmulhuw %xmm2, %xmm02478; SSE2-NEXT:    psrlw $15, %xmm02479; SSE2-NEXT:    retq2480;2481; SSE41-LABEL: and_mulhuw_v4i16_shift31:2482; SSE41:       # %bb.0:2483; SSE41-NEXT:    pxor %xmm4, %xmm42484; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1,2,3],xmm3[4],xmm4[5,6,7]2485; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1,2,3],xmm2[4],xmm4[5,6,7]2486; SSE41-NEXT:    packusdw %xmm3, %xmm22487; SSE41-NEXT:    packusdw %xmm4, %xmm22488; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0],xmm4[1,2,3],xmm1[4],xmm4[5,6,7]2489; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm4[1,2,3],xmm0[4],xmm4[5,6,7]2490; SSE41-NEXT:    packusdw %xmm1, %xmm02491; SSE41-NEXT:    packusdw %xmm4, %xmm02492; SSE41-NEXT:    pmulhuw %xmm2, %xmm02493; SSE41-NEXT:    psrlw $15, %xmm02494; SSE41-NEXT:    retq2495;2496; AVX2-LABEL: and_mulhuw_v4i16_shift31:2497; AVX2:       # %bb.0:2498; AVX2-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm02499; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm12500; AVX2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15]2501; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm12502; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm02503; AVX2-NEXT:    vpackusdw %xmm0, %xmm0, %xmm02504; AVX2-NEXT:    vpsrlw $15, %xmm0, %xmm02505; AVX2-NEXT:    vzeroupper2506; AVX2-NEXT:    retq2507;2508; AVX512-LABEL: and_mulhuw_v4i16_shift31:2509; AVX512:       # %bb.0:2510; AVX512-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm02511; AVX512-NEXT:    vpmovqw %zmm0, %xmm02512; AVX512-NEXT:    vpsrlw $15, %xmm0, %xmm02513; AVX512-NEXT:    vzeroupper2514; AVX512-NEXT:    retq2515  %a1 = and <4 x i64> %a, <i64 65535, i64 65535, i64 65535, i64 65535>2516  %b1 = and <4 x i64> %b, <i64 65535, i64 65535, i64 65535, i64 65535>2517  %c = mul <4 x i64> %a1, %b12518  %d = lshr <4 x i64> %c, splat (i64 31)2519  %e = trunc <4 x i64> %d to <4 x i16>2520  ret <4 x i16> %e2521}2522 2523define <8 x i16> @lshr_mulhuw_v8i16_shift17(<8 x i32> %a, <8 x i32> %b) {2524; SSE2-LABEL: lshr_mulhuw_v8i16_shift17:2525; SSE2:       # %bb.0:2526; SSE2-NEXT:    psrad $16, %xmm32527; SSE2-NEXT:    psrad $16, %xmm22528; SSE2-NEXT:    packssdw %xmm3, %xmm22529; SSE2-NEXT:    psrad $16, %xmm12530; SSE2-NEXT:    psrad $16, %xmm02531; SSE2-NEXT:    packssdw %xmm1, %xmm02532; SSE2-NEXT:    pmulhuw %xmm2, %xmm02533; SSE2-NEXT:    psrlw $1, %xmm02534; SSE2-NEXT:    retq2535;2536; SSE41-LABEL: lshr_mulhuw_v8i16_shift17:2537; SSE41:       # %bb.0:2538; SSE41-NEXT:    psrld $16, %xmm12539; SSE41-NEXT:    psrld $16, %xmm02540; SSE41-NEXT:    packusdw %xmm1, %xmm02541; SSE41-NEXT:    psrld $16, %xmm32542; SSE41-NEXT:    psrld $16, %xmm22543; SSE41-NEXT:    packusdw %xmm3, %xmm22544; SSE41-NEXT:    pmulhuw %xmm2, %xmm02545; SSE41-NEXT:    psrlw $1, %xmm02546; SSE41-NEXT:    retq2547;2548; AVX2-LABEL: lshr_mulhuw_v8i16_shift17:2549; AVX2:       # %bb.0:2550; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm02551; AVX2-NEXT:    vpsrld $16, %ymm1, %ymm12552; AVX2-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm02553; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm12554; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm02555; AVX2-NEXT:    vpsrlw $1, %xmm0, %xmm02556; AVX2-NEXT:    vzeroupper2557; AVX2-NEXT:    retq2558;2559; AVX512-LABEL: lshr_mulhuw_v8i16_shift17:2560; AVX512:       # %bb.0:2561; AVX512-NEXT:    vpsrld $16, %ymm0, %ymm02562; AVX512-NEXT:    vpsrld $16, %ymm1, %ymm12563; AVX512-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm02564; AVX512-NEXT:    vpmovdw %zmm0, %ymm02565; AVX512-NEXT:    vpsrlw $1, %xmm0, %xmm02566; AVX512-NEXT:    vzeroupper2567; AVX512-NEXT:    retq2568  %a1 = lshr <8 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>2569  %b1 = lshr <8 x i32> %b, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>2570  %c = mul <8 x i32> %a1, %b12571  %d = lshr <8 x i32> %c, splat (i32 17)2572  %e = trunc <8 x i32> %d to <8 x i16>2573  ret <8 x i16> %e2574}2575 2576define <8 x i16> @lshr_mulhuw_v8i16_shift24(<8 x i32> %a, <8 x i32> %b) {2577; SSE2-LABEL: lshr_mulhuw_v8i16_shift24:2578; SSE2:       # %bb.0:2579; SSE2-NEXT:    psrad $16, %xmm32580; SSE2-NEXT:    psrad $16, %xmm22581; SSE2-NEXT:    packssdw %xmm3, %xmm22582; SSE2-NEXT:    psrad $16, %xmm12583; SSE2-NEXT:    psrad $16, %xmm02584; SSE2-NEXT:    packssdw %xmm1, %xmm02585; SSE2-NEXT:    pmulhuw %xmm2, %xmm02586; SSE2-NEXT:    psrlw $8, %xmm02587; SSE2-NEXT:    retq2588;2589; SSE41-LABEL: lshr_mulhuw_v8i16_shift24:2590; SSE41:       # %bb.0:2591; SSE41-NEXT:    psrld $16, %xmm12592; SSE41-NEXT:    psrld $16, %xmm02593; SSE41-NEXT:    packusdw %xmm1, %xmm02594; SSE41-NEXT:    psrld $16, %xmm32595; SSE41-NEXT:    psrld $16, %xmm22596; SSE41-NEXT:    packusdw %xmm3, %xmm22597; SSE41-NEXT:    pmulhuw %xmm2, %xmm02598; SSE41-NEXT:    psrlw $8, %xmm02599; SSE41-NEXT:    retq2600;2601; AVX2-LABEL: lshr_mulhuw_v8i16_shift24:2602; AVX2:       # %bb.0:2603; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm02604; AVX2-NEXT:    vpsrld $16, %ymm1, %ymm12605; AVX2-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm02606; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm12607; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm02608; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm02609; AVX2-NEXT:    vzeroupper2610; AVX2-NEXT:    retq2611;2612; AVX512-LABEL: lshr_mulhuw_v8i16_shift24:2613; AVX512:       # %bb.0:2614; AVX512-NEXT:    vpsrld $16, %ymm0, %ymm02615; AVX512-NEXT:    vpsrld $16, %ymm1, %ymm12616; AVX512-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm02617; AVX512-NEXT:    vpmovdw %zmm0, %ymm02618; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm02619; AVX512-NEXT:    vzeroupper2620; AVX512-NEXT:    retq2621  %a1 = lshr <8 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>2622  %b1 = lshr <8 x i32> %b, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>2623  %c = mul <8 x i32> %a1, %b12624  %d = lshr <8 x i32> %c, splat (i32 24)2625  %e = trunc <8 x i32> %d to <8 x i16>2626  ret <8 x i16> %e2627}2628 2629define <8 x i16> @lshr_mulhuw_v8i16_shift31(<8 x i32> %a, <8 x i32> %b) {2630; SSE2-LABEL: lshr_mulhuw_v8i16_shift31:2631; SSE2:       # %bb.0:2632; SSE2-NEXT:    psrad $16, %xmm32633; SSE2-NEXT:    psrad $16, %xmm22634; SSE2-NEXT:    packssdw %xmm3, %xmm22635; SSE2-NEXT:    psrad $16, %xmm12636; SSE2-NEXT:    psrad $16, %xmm02637; SSE2-NEXT:    packssdw %xmm1, %xmm02638; SSE2-NEXT:    pmulhuw %xmm2, %xmm02639; SSE2-NEXT:    psrlw $15, %xmm02640; SSE2-NEXT:    retq2641;2642; SSE41-LABEL: lshr_mulhuw_v8i16_shift31:2643; SSE41:       # %bb.0:2644; SSE41-NEXT:    psrld $16, %xmm12645; SSE41-NEXT:    psrld $16, %xmm02646; SSE41-NEXT:    packusdw %xmm1, %xmm02647; SSE41-NEXT:    psrld $16, %xmm32648; SSE41-NEXT:    psrld $16, %xmm22649; SSE41-NEXT:    packusdw %xmm3, %xmm22650; SSE41-NEXT:    pmulhuw %xmm2, %xmm02651; SSE41-NEXT:    psrlw $15, %xmm02652; SSE41-NEXT:    retq2653;2654; AVX2-LABEL: lshr_mulhuw_v8i16_shift31:2655; AVX2:       # %bb.0:2656; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm02657; AVX2-NEXT:    vpsrld $16, %ymm1, %ymm12658; AVX2-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm02659; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm12660; AVX2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm02661; AVX2-NEXT:    vpsrlw $15, %xmm0, %xmm02662; AVX2-NEXT:    vzeroupper2663; AVX2-NEXT:    retq2664;2665; AVX512-LABEL: lshr_mulhuw_v8i16_shift31:2666; AVX512:       # %bb.0:2667; AVX512-NEXT:    vpsrld $16, %ymm0, %ymm02668; AVX512-NEXT:    vpsrld $16, %ymm1, %ymm12669; AVX512-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm02670; AVX512-NEXT:    vpmovdw %zmm0, %ymm02671; AVX512-NEXT:    vpsrlw $15, %xmm0, %xmm02672; AVX512-NEXT:    vzeroupper2673; AVX512-NEXT:    retq2674  %a1 = lshr <8 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>2675  %b1 = lshr <8 x i32> %b, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>2676  %c = mul <8 x i32> %a1, %b12677  %d = lshr <8 x i32> %c, splat (i32 31)2678  %e = trunc <8 x i32> %d to <8 x i16>2679  ret <8 x i16> %e2680}2681 2682define <16 x i16> @and_mulhuw_v16i16_shift17(<16 x i32> %a, <16 x i32> %b) {2683; SSE2-LABEL: and_mulhuw_v16i16_shift17:2684; SSE2:       # %bb.0:2685; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [32767,32767,32767,32767]2686; SSE2-NEXT:    pand %xmm8, %xmm32687; SSE2-NEXT:    pand %xmm8, %xmm22688; SSE2-NEXT:    packssdw %xmm3, %xmm22689; SSE2-NEXT:    pand %xmm8, %xmm12690; SSE2-NEXT:    pand %xmm8, %xmm02691; SSE2-NEXT:    packssdw %xmm1, %xmm02692; SSE2-NEXT:    pand %xmm8, %xmm72693; SSE2-NEXT:    pand %xmm8, %xmm62694; SSE2-NEXT:    packssdw %xmm7, %xmm62695; SSE2-NEXT:    pmulhw %xmm2, %xmm62696; SSE2-NEXT:    pand %xmm8, %xmm52697; SSE2-NEXT:    pand %xmm4, %xmm82698; SSE2-NEXT:    packssdw %xmm5, %xmm82699; SSE2-NEXT:    pmulhw %xmm8, %xmm02700; SSE2-NEXT:    psrlw $1, %xmm02701; SSE2-NEXT:    psrlw $1, %xmm62702; SSE2-NEXT:    movdqa %xmm6, %xmm12703; SSE2-NEXT:    retq2704;2705; SSE41-LABEL: and_mulhuw_v16i16_shift17:2706; SSE41:       # %bb.0:2707; SSE41-NEXT:    pmovsxwd {{.*#+}} xmm8 = [32767,32767,32767,32767]2708; SSE41-NEXT:    pand %xmm8, %xmm32709; SSE41-NEXT:    pand %xmm8, %xmm22710; SSE41-NEXT:    packusdw %xmm3, %xmm22711; SSE41-NEXT:    pand %xmm8, %xmm12712; SSE41-NEXT:    pand %xmm8, %xmm02713; SSE41-NEXT:    packusdw %xmm1, %xmm02714; SSE41-NEXT:    pand %xmm8, %xmm72715; SSE41-NEXT:    pand %xmm8, %xmm62716; SSE41-NEXT:    packusdw %xmm7, %xmm62717; SSE41-NEXT:    pmulhw %xmm2, %xmm62718; SSE41-NEXT:    pand %xmm8, %xmm52719; SSE41-NEXT:    pand %xmm4, %xmm82720; SSE41-NEXT:    packusdw %xmm5, %xmm82721; SSE41-NEXT:    pmulhw %xmm8, %xmm02722; SSE41-NEXT:    psrlw $1, %xmm02723; SSE41-NEXT:    psrlw $1, %xmm62724; SSE41-NEXT:    movdqa %xmm6, %xmm12725; SSE41-NEXT:    retq2726;2727; AVX2-LABEL: and_mulhuw_v16i16_shift17:2728; AVX2:       # %bb.0:2729; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm4 = [32767,32767,32767,32767,32767,32767,32767,32767]2730; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm02731; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm12732; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm22733; AVX2-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm02734; AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm22735; AVX2-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm12736; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm02737; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]2738; AVX2-NEXT:    vpsrlw $1, %ymm0, %ymm02739; AVX2-NEXT:    retq2740;2741; AVX512F-LABEL: and_mulhuw_v16i16_shift17:2742; AVX512F:       # %bb.0:2743; AVX512F-NEXT:    vpmovdw %zmm1, %ymm12744; AVX512F-NEXT:    vpbroadcastw {{.*#+}} ymm2 = [32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767]2745; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm12746; AVX512F-NEXT:    vpmovdw %zmm0, %ymm02747; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm02748; AVX512F-NEXT:    vpmulhw %ymm1, %ymm0, %ymm02749; AVX512F-NEXT:    vpsrlw $1, %ymm0, %ymm02750; AVX512F-NEXT:    retq2751;2752; AVX512BW-LABEL: and_mulhuw_v16i16_shift17:2753; AVX512BW:       # %bb.0:2754; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} zmm2 = [32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767]2755; AVX512BW-NEXT:    vpandd %zmm2, %zmm0, %zmm02756; AVX512BW-NEXT:    vpandd %zmm2, %zmm1, %zmm12757; AVX512BW-NEXT:    vpmulhuw %zmm1, %zmm0, %zmm02758; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm02759; AVX512BW-NEXT:    vpsrlw $1, %ymm0, %ymm02760; AVX512BW-NEXT:    retq2761  %a1 = and <16 x i32> %a, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>2762  %b1 = and <16 x i32> %b, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>2763  %c = mul <16 x i32> %a1, %b12764  %d = lshr <16 x i32> %c, splat (i32 17)2765  %e = trunc <16 x i32> %d to <16 x i16>2766  ret <16 x i16> %e2767}2768 2769define <16 x i16> @and_mulhuw_v16i16_shift24(<16 x i32> %a, <16 x i32> %b) {2770; SSE2-LABEL: and_mulhuw_v16i16_shift24:2771; SSE2:       # %bb.0:2772; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [32767,32767,32767,32767]2773; SSE2-NEXT:    pand %xmm8, %xmm32774; SSE2-NEXT:    pand %xmm8, %xmm22775; SSE2-NEXT:    packssdw %xmm3, %xmm22776; SSE2-NEXT:    pand %xmm8, %xmm12777; SSE2-NEXT:    pand %xmm8, %xmm02778; SSE2-NEXT:    packssdw %xmm1, %xmm02779; SSE2-NEXT:    pand %xmm8, %xmm72780; SSE2-NEXT:    pand %xmm8, %xmm62781; SSE2-NEXT:    packssdw %xmm7, %xmm62782; SSE2-NEXT:    pmulhw %xmm2, %xmm62783; SSE2-NEXT:    pand %xmm8, %xmm52784; SSE2-NEXT:    pand %xmm4, %xmm82785; SSE2-NEXT:    packssdw %xmm5, %xmm82786; SSE2-NEXT:    pmulhw %xmm8, %xmm02787; SSE2-NEXT:    psrlw $8, %xmm02788; SSE2-NEXT:    psrlw $8, %xmm62789; SSE2-NEXT:    movdqa %xmm6, %xmm12790; SSE2-NEXT:    retq2791;2792; SSE41-LABEL: and_mulhuw_v16i16_shift24:2793; SSE41:       # %bb.0:2794; SSE41-NEXT:    pmovsxwd {{.*#+}} xmm8 = [32767,32767,32767,32767]2795; SSE41-NEXT:    pand %xmm8, %xmm32796; SSE41-NEXT:    pand %xmm8, %xmm22797; SSE41-NEXT:    packusdw %xmm3, %xmm22798; SSE41-NEXT:    pand %xmm8, %xmm12799; SSE41-NEXT:    pand %xmm8, %xmm02800; SSE41-NEXT:    packusdw %xmm1, %xmm02801; SSE41-NEXT:    pand %xmm8, %xmm72802; SSE41-NEXT:    pand %xmm8, %xmm62803; SSE41-NEXT:    packusdw %xmm7, %xmm62804; SSE41-NEXT:    pmulhw %xmm2, %xmm62805; SSE41-NEXT:    pand %xmm8, %xmm52806; SSE41-NEXT:    pand %xmm4, %xmm82807; SSE41-NEXT:    packusdw %xmm5, %xmm82808; SSE41-NEXT:    pmulhw %xmm8, %xmm02809; SSE41-NEXT:    psrlw $8, %xmm02810; SSE41-NEXT:    psrlw $8, %xmm62811; SSE41-NEXT:    movdqa %xmm6, %xmm12812; SSE41-NEXT:    retq2813;2814; AVX2-LABEL: and_mulhuw_v16i16_shift24:2815; AVX2:       # %bb.0:2816; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm4 = [32767,32767,32767,32767,32767,32767,32767,32767]2817; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm02818; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm12819; AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm22820; AVX2-NEXT:    vpmulhuw %ymm2, %ymm0, %ymm02821; AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm22822; AVX2-NEXT:    vpmulhuw %ymm2, %ymm1, %ymm12823; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm02824; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]2825; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm02826; AVX2-NEXT:    retq2827;2828; AVX512F-LABEL: and_mulhuw_v16i16_shift24:2829; AVX512F:       # %bb.0:2830; AVX512F-NEXT:    vpmovdw %zmm1, %ymm12831; AVX512F-NEXT:    vpbroadcastw {{.*#+}} ymm2 = [32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767]2832; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm12833; AVX512F-NEXT:    vpmovdw %zmm0, %ymm02834; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm02835; AVX512F-NEXT:    vpmulhw %ymm1, %ymm0, %ymm02836; AVX512F-NEXT:    vpsrlw $8, %ymm0, %ymm02837; AVX512F-NEXT:    retq2838;2839; AVX512BW-LABEL: and_mulhuw_v16i16_shift24:2840; AVX512BW:       # %bb.0:2841; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} zmm2 = [32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767]2842; AVX512BW-NEXT:    vpandd %zmm2, %zmm0, %zmm02843; AVX512BW-NEXT:    vpandd %zmm2, %zmm1, %zmm12844; AVX512BW-NEXT:    vpmulhuw %zmm1, %zmm0, %zmm02845; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm02846; AVX512BW-NEXT:    vpsrlw $8, %ymm0, %ymm02847; AVX512BW-NEXT:    retq2848  %a1 = and <16 x i32> %a, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>2849  %b1 = and <16 x i32> %b, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>2850  %c = mul <16 x i32> %a1, %b12851  %d = lshr <16 x i32> %c, splat (i32 24)2852  %e = trunc <16 x i32> %d to <16 x i16>2853  ret <16 x i16> %e2854}2855 2856define <16 x i16> @and_mulhuw_v16i16_shift31(<16 x i32> %a, <16 x i32> %b) {2857; SSE-LABEL: and_mulhuw_v16i16_shift31:2858; SSE:       # %bb.0:2859; SSE-NEXT:    xorps %xmm0, %xmm02860; SSE-NEXT:    xorps %xmm1, %xmm12861; SSE-NEXT:    retq2862;2863; AVX-LABEL: and_mulhuw_v16i16_shift31:2864; AVX:       # %bb.0:2865; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm02866; AVX-NEXT:    retq2867  %a1 = and <16 x i32> %a, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>2868  %b1 = and <16 x i32> %b, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>2869  %c = mul <16 x i32> %a1, %b12870  %d = lshr <16 x i32> %c, splat (i32 31)2871  %e = trunc <16 x i32> %d to <16 x i16>2872  ret <16 x i16> %e2873}2874