brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE23; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE34; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE415; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX16; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX27; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F8; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F9; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW10; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW11 12declare {<1 x i32>, <1 x i1>} @llvm.umul.with.overflow.v1i32(<1 x i32>, <1 x i32>)13declare {<2 x i32>, <2 x i1>} @llvm.umul.with.overflow.v2i32(<2 x i32>, <2 x i32>)14declare {<3 x i32>, <3 x i1>} @llvm.umul.with.overflow.v3i32(<3 x i32>, <3 x i32>)15declare {<4 x i32>, <4 x i1>} @llvm.umul.with.overflow.v4i32(<4 x i32>, <4 x i32>)16declare {<6 x i32>, <6 x i1>} @llvm.umul.with.overflow.v6i32(<6 x i32>, <6 x i32>)17declare {<8 x i32>, <8 x i1>} @llvm.umul.with.overflow.v8i32(<8 x i32>, <8 x i32>)18declare {<16 x i32>, <16 x i1>} @llvm.umul.with.overflow.v16i32(<16 x i32>, <16 x i32>)19 20declare {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8>, <16 x i8>)21declare {<32 x i8>, <32 x i1>} @llvm.umul.with.overflow.v32i8(<32 x i8>, <32 x i8>)22declare {<64 x i8>, <64 x i1>} @llvm.umul.with.overflow.v64i8(<64 x i8>, <64 x i8>)23declare {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16>, <8 x i16>)24declare {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64>, <2 x i64>)25 26declare {<4 x i24>, <4 x i1>} @llvm.umul.with.overflow.v4i24(<4 x i24>, <4 x i24>)27declare {<4 x i1>, <4 x i1>} @llvm.umul.with.overflow.v4i1(<4 x i1>, <4 x i1>)28declare {<2 x i128>, <2 x i1>} @llvm.umul.with.overflow.v2i128(<2 x i128>, <2 x i128>)29 30define <1 x i32> @umulo_v1i32(<1 x i32> %a0, <1 x i32> %a1, ptr %p2) nounwind {31; CHECK-LABEL: umulo_v1i32:32; CHECK:       # %bb.0:33; CHECK-NEXT:    movq %rdx, %rcx34; CHECK-NEXT:    movl %edi, %eax35; CHECK-NEXT:    xorl %edi, %edi36; CHECK-NEXT:    mull %esi37; CHECK-NEXT:    seto %dil38; CHECK-NEXT:    negl %edi39; CHECK-NEXT:    movl %eax, (%rcx)40; CHECK-NEXT:    movl %edi, %eax41; CHECK-NEXT:    retq42  %t = call {<1 x i32>, <1 x i1>} @llvm.umul.with.overflow.v1i32(<1 x i32> %a0, <1 x i32> %a1)43  %val = extractvalue {<1 x i32>, <1 x i1>} %t, 044  %obit = extractvalue {<1 x i32>, <1 x i1>} %t, 145  %res = sext <1 x i1> %obit to <1 x i32>46  store <1 x i32> %val, ptr %p247  ret <1 x i32> %res48}49 50define <2 x i32> @umulo_v2i32(<2 x i32> %a0, <2 x i32> %a1, ptr %p2) nounwind {51; SSE2-LABEL: umulo_v2i32:52; SSE2:       # %bb.0:53; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]54; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,1,1,3]55; SSE2-NEXT:    pmuludq %xmm1, %xmm256; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]57; SSE2-NEXT:    pxor %xmm1, %xmm158; SSE2-NEXT:    pcmpeqd %xmm0, %xmm159; SSE2-NEXT:    pcmpeqd %xmm0, %xmm060; SSE2-NEXT:    pxor %xmm1, %xmm061; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]62; SSE2-NEXT:    movq %xmm1, (%rdi)63; SSE2-NEXT:    retq64;65; SSSE3-LABEL: umulo_v2i32:66; SSSE3:       # %bb.0:67; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]68; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,1,1,3]69; SSSE3-NEXT:    pmuludq %xmm1, %xmm270; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]71; SSSE3-NEXT:    pxor %xmm1, %xmm172; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm173; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm074; SSSE3-NEXT:    pxor %xmm1, %xmm075; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]76; SSSE3-NEXT:    movq %xmm1, (%rdi)77; SSSE3-NEXT:    retq78;79; SSE41-LABEL: umulo_v2i32:80; SSE41:       # %bb.0:81; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero82; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero83; SSE41-NEXT:    pmuludq %xmm1, %xmm284; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]85; SSE41-NEXT:    pxor %xmm1, %xmm186; SSE41-NEXT:    pcmpeqd %xmm0, %xmm187; SSE41-NEXT:    pcmpeqd %xmm0, %xmm088; SSE41-NEXT:    pxor %xmm1, %xmm089; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]90; SSE41-NEXT:    movq %xmm1, (%rdi)91; SSE41-NEXT:    retq92;93; AVX-LABEL: umulo_v2i32:94; AVX:       # %bb.0:95; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero96; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero97; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm198; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]99; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2100; AVX-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0101; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2102; AVX-NEXT:    vpxor %xmm2, %xmm0, %xmm0103; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]104; AVX-NEXT:    vmovq %xmm1, (%rdi)105; AVX-NEXT:    retq106;107; AVX512-LABEL: umulo_v2i32:108; AVX512:       # %bb.0:109; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero110; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero111; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0112; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]113; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]114; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2115; AVX512-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0116; AVX512-NEXT:    vpternlogq {{.*#+}} xmm0 = ~xmm0117; AVX512-NEXT:    vmovq %xmm1, (%rdi)118; AVX512-NEXT:    retq119  %t = call {<2 x i32>, <2 x i1>} @llvm.umul.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)120  %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0121  %obit = extractvalue {<2 x i32>, <2 x i1>} %t, 1122  %res = sext <2 x i1> %obit to <2 x i32>123  store <2 x i32> %val, ptr %p2124  ret <2 x i32> %res125}126 127define <3 x i32> @umulo_v3i32(<3 x i32> %a0, <3 x i32> %a1, ptr %p2) nounwind {128; SSE2-LABEL: umulo_v3i32:129; SSE2:       # %bb.0:130; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]131; SSE2-NEXT:    pmuludq %xmm1, %xmm0132; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]133; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]134; SSE2-NEXT:    pmuludq %xmm2, %xmm4135; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]136; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]137; SSE2-NEXT:    pxor %xmm2, %xmm2138; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2139; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1140; SSE2-NEXT:    pxor %xmm2, %xmm1141; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]142; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]143; SSE2-NEXT:    movd %xmm2, 8(%rdi)144; SSE2-NEXT:    movq %xmm0, (%rdi)145; SSE2-NEXT:    movdqa %xmm1, %xmm0146; SSE2-NEXT:    retq147;148; SSSE3-LABEL: umulo_v3i32:149; SSSE3:       # %bb.0:150; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]151; SSSE3-NEXT:    pmuludq %xmm1, %xmm0152; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]153; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]154; SSSE3-NEXT:    pmuludq %xmm2, %xmm4155; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]156; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]157; SSSE3-NEXT:    pxor %xmm2, %xmm2158; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2159; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1160; SSSE3-NEXT:    pxor %xmm2, %xmm1161; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]162; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]163; SSSE3-NEXT:    movd %xmm2, 8(%rdi)164; SSSE3-NEXT:    movq %xmm0, (%rdi)165; SSSE3-NEXT:    movdqa %xmm1, %xmm0166; SSSE3-NEXT:    retq167;168; SSE41-LABEL: umulo_v3i32:169; SSE41:       # %bb.0:170; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]171; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]172; SSE41-NEXT:    pmuludq %xmm2, %xmm3173; SSE41-NEXT:    movdqa %xmm0, %xmm2174; SSE41-NEXT:    pmuludq %xmm1, %xmm2175; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]176; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]177; SSE41-NEXT:    pxor %xmm3, %xmm3178; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3179; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2180; SSE41-NEXT:    pxor %xmm3, %xmm2181; SSE41-NEXT:    pmulld %xmm1, %xmm0182; SSE41-NEXT:    pextrd $2, %xmm0, 8(%rdi)183; SSE41-NEXT:    movq %xmm0, (%rdi)184; SSE41-NEXT:    movdqa %xmm2, %xmm0185; SSE41-NEXT:    retq186;187; AVX1-LABEL: umulo_v3i32:188; AVX1:       # %bb.0:189; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]190; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]191; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2192; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3193; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]194; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]195; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3196; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2197; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3198; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2199; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0200; AVX1-NEXT:    vpextrd $2, %xmm0, 8(%rdi)201; AVX1-NEXT:    vmovq %xmm0, (%rdi)202; AVX1-NEXT:    vmovdqa %xmm2, %xmm0203; AVX1-NEXT:    retq204;205; AVX2-LABEL: umulo_v3i32:206; AVX2:       # %bb.0:207; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]208; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]209; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2210; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3211; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]212; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]213; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3214; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2215; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3216; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2217; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0218; AVX2-NEXT:    vpextrd $2, %xmm0, 8(%rdi)219; AVX2-NEXT:    vmovq %xmm0, (%rdi)220; AVX2-NEXT:    vmovdqa %xmm2, %xmm0221; AVX2-NEXT:    retq222;223; AVX512-LABEL: umulo_v3i32:224; AVX512:       # %bb.0:225; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2226; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]227; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]228; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3229; AVX512-NEXT:    vpmovsxbd {{.*#+}} xmm4 = [1,5,3,7]230; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4231; AVX512-NEXT:    vptestmd %xmm4, %xmm4, %k1232; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1233; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0234; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}235; AVX512-NEXT:    vpextrd $2, %xmm1, 8(%rdi)236; AVX512-NEXT:    vmovq %xmm1, (%rdi)237; AVX512-NEXT:    retq238  %t = call {<3 x i32>, <3 x i1>} @llvm.umul.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)239  %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0240  %obit = extractvalue {<3 x i32>, <3 x i1>} %t, 1241  %res = sext <3 x i1> %obit to <3 x i32>242  store <3 x i32> %val, ptr %p2243  ret <3 x i32> %res244}245 246define <4 x i32> @umulo_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %p2) nounwind {247; SSE2-LABEL: umulo_v4i32:248; SSE2:       # %bb.0:249; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]250; SSE2-NEXT:    pmuludq %xmm1, %xmm0251; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]252; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]253; SSE2-NEXT:    pmuludq %xmm2, %xmm4254; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]255; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]256; SSE2-NEXT:    pxor %xmm2, %xmm2257; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2258; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1259; SSE2-NEXT:    pxor %xmm2, %xmm1260; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]261; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]262; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]263; SSE2-NEXT:    movdqa %xmm0, (%rdi)264; SSE2-NEXT:    movdqa %xmm1, %xmm0265; SSE2-NEXT:    retq266;267; SSSE3-LABEL: umulo_v4i32:268; SSSE3:       # %bb.0:269; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]270; SSSE3-NEXT:    pmuludq %xmm1, %xmm0271; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]272; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]273; SSSE3-NEXT:    pmuludq %xmm2, %xmm4274; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,3,2,3]275; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]276; SSSE3-NEXT:    pxor %xmm2, %xmm2277; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2278; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1279; SSSE3-NEXT:    pxor %xmm2, %xmm1280; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]281; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]282; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]283; SSSE3-NEXT:    movdqa %xmm0, (%rdi)284; SSSE3-NEXT:    movdqa %xmm1, %xmm0285; SSSE3-NEXT:    retq286;287; SSE41-LABEL: umulo_v4i32:288; SSE41:       # %bb.0:289; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]290; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]291; SSE41-NEXT:    pmuludq %xmm2, %xmm3292; SSE41-NEXT:    movdqa %xmm0, %xmm2293; SSE41-NEXT:    pmuludq %xmm1, %xmm2294; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]295; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]296; SSE41-NEXT:    pxor %xmm3, %xmm3297; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3298; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2299; SSE41-NEXT:    pxor %xmm3, %xmm2300; SSE41-NEXT:    pmulld %xmm1, %xmm0301; SSE41-NEXT:    movdqa %xmm0, (%rdi)302; SSE41-NEXT:    movdqa %xmm2, %xmm0303; SSE41-NEXT:    retq304;305; AVX1-LABEL: umulo_v4i32:306; AVX1:       # %bb.0:307; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]308; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]309; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2310; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3311; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]312; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]313; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3314; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2315; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3316; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2317; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0318; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)319; AVX1-NEXT:    vmovdqa %xmm2, %xmm0320; AVX1-NEXT:    retq321;322; AVX2-LABEL: umulo_v4i32:323; AVX2:       # %bb.0:324; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]325; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]326; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2327; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm3328; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]329; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]330; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3331; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm2332; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3333; AVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2334; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0335; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)336; AVX2-NEXT:    vmovdqa %xmm2, %xmm0337; AVX2-NEXT:    retq338;339; AVX512-LABEL: umulo_v4i32:340; AVX512:       # %bb.0:341; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm2342; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]343; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]344; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3345; AVX512-NEXT:    vpmovsxbd {{.*#+}} xmm4 = [1,5,3,7]346; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm4347; AVX512-NEXT:    vptestmd %xmm4, %xmm4, %k1348; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm1349; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0350; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}351; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)352; AVX512-NEXT:    retq353  %t = call {<4 x i32>, <4 x i1>} @llvm.umul.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a1)354  %val = extractvalue {<4 x i32>, <4 x i1>} %t, 0355  %obit = extractvalue {<4 x i32>, <4 x i1>} %t, 1356  %res = sext <4 x i1> %obit to <4 x i32>357  store <4 x i32> %val, ptr %p2358  ret <4 x i32> %res359}360 361define <6 x i32> @umulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, ptr %p2) nounwind {362; SSE2-LABEL: umulo_v6i32:363; SSE2:       # %bb.0:364; SSE2-NEXT:    movq %rdi, %rax365; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero366; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero367; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]368; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero369; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero370; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]371; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]372; SSE2-NEXT:    movd %r8d, %xmm0373; SSE2-NEXT:    movd %ecx, %xmm1374; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]375; SSE2-NEXT:    movd %edx, %xmm0376; SSE2-NEXT:    movd %esi, %xmm3377; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]378; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0]379; SSE2-NEXT:    movd %r9d, %xmm1380; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx381; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero382; SSE2-NEXT:    pmuludq %xmm1, %xmm0383; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]384; SSE2-NEXT:    pmuludq %xmm2, %xmm3385; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]386; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]387; SSE2-NEXT:    pmuludq %xmm4, %xmm2388; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3]389; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]390; SSE2-NEXT:    pxor %xmm4, %xmm4391; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1392; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5393; SSE2-NEXT:    pxor %xmm5, %xmm1394; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]395; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]396; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]397; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero398; SSE2-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero399; SSE2-NEXT:    pmuludq %xmm2, %xmm6400; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3]401; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,3,2,3]402; SSE2-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]403; SSE2-NEXT:    pcmpeqd %xmm4, %xmm7404; SSE2-NEXT:    pxor %xmm5, %xmm7405; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]406; SSE2-NEXT:    movq %xmm0, 16(%rcx)407; SSE2-NEXT:    movdqa %xmm3, (%rcx)408; SSE2-NEXT:    movq %xmm7, 16(%rdi)409; SSE2-NEXT:    movdqa %xmm1, (%rdi)410; SSE2-NEXT:    retq411;412; SSSE3-LABEL: umulo_v6i32:413; SSSE3:       # %bb.0:414; SSSE3-NEXT:    movq %rdi, %rax415; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero416; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero417; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]418; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero419; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero420; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]421; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]422; SSSE3-NEXT:    movd %r8d, %xmm0423; SSSE3-NEXT:    movd %ecx, %xmm1424; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]425; SSSE3-NEXT:    movd %edx, %xmm0426; SSSE3-NEXT:    movd %esi, %xmm3427; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]428; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0]429; SSSE3-NEXT:    movd %r9d, %xmm1430; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx431; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero432; SSSE3-NEXT:    pmuludq %xmm1, %xmm0433; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]434; SSSE3-NEXT:    pmuludq %xmm2, %xmm3435; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]436; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]437; SSSE3-NEXT:    pmuludq %xmm4, %xmm2438; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,3,2,3]439; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]440; SSSE3-NEXT:    pxor %xmm4, %xmm4441; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1442; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5443; SSSE3-NEXT:    pxor %xmm5, %xmm1444; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]445; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]446; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]447; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero448; SSSE3-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero449; SSSE3-NEXT:    pmuludq %xmm2, %xmm6450; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3]451; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,3,2,3]452; SSSE3-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]453; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm7454; SSSE3-NEXT:    pxor %xmm5, %xmm7455; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]456; SSSE3-NEXT:    movq %xmm0, 16(%rcx)457; SSSE3-NEXT:    movdqa %xmm3, (%rcx)458; SSSE3-NEXT:    movq %xmm7, 16(%rdi)459; SSSE3-NEXT:    movdqa %xmm1, (%rdi)460; SSSE3-NEXT:    retq461;462; SSE41-LABEL: umulo_v6i32:463; SSE41:       # %bb.0:464; SSE41-NEXT:    movq %rdi, %rax465; SSE41-NEXT:    movl {{[0-9]+}}(%rsp), %edi466; SSE41-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero467; SSE41-NEXT:    movd %r9d, %xmm1468; SSE41-NEXT:    movdqa %xmm1, %xmm0469; SSE41-NEXT:    pmuludq %xmm2, %xmm1470; SSE41-NEXT:    pinsrd $1, %edi, %xmm2471; SSE41-NEXT:    movl {{[0-9]+}}(%rsp), %r9d472; SSE41-NEXT:    pinsrd $1, %r9d, %xmm0473; SSE41-NEXT:    pmulld %xmm2, %xmm0474; SSE41-NEXT:    movd %esi, %xmm2475; SSE41-NEXT:    pinsrd $1, %edx, %xmm2476; SSE41-NEXT:    pinsrd $2, %ecx, %xmm2477; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero478; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3479; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm3480; SSE41-NEXT:    movdqa %xmm3, %xmm4481; SSE41-NEXT:    pmuludq %xmm2, %xmm3482; SSE41-NEXT:    pinsrd $3, %r8d, %xmm2483; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm4484; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx485; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]486; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]487; SSE41-NEXT:    pmuludq %xmm5, %xmm6488; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]489; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm6[2,3],xmm3[4,5],xmm6[6,7]490; SSE41-NEXT:    pxor %xmm5, %xmm5491; SSE41-NEXT:    pcmpeqd %xmm5, %xmm3492; SSE41-NEXT:    pcmpeqd %xmm6, %xmm6493; SSE41-NEXT:    pxor %xmm6, %xmm3494; SSE41-NEXT:    movd %edi, %xmm7495; SSE41-NEXT:    movd %r9d, %xmm8496; SSE41-NEXT:    pmuludq %xmm7, %xmm8497; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]498; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm8[2,3],xmm1[4,5],xmm8[6,7]499; SSE41-NEXT:    pcmpeqd %xmm5, %xmm1500; SSE41-NEXT:    pxor %xmm6, %xmm1501; SSE41-NEXT:    pmulld %xmm2, %xmm4502; SSE41-NEXT:    movq %xmm0, 16(%rcx)503; SSE41-NEXT:    movdqa %xmm4, (%rcx)504; SSE41-NEXT:    movq %xmm1, 16(%rax)505; SSE41-NEXT:    movdqa %xmm3, (%rax)506; SSE41-NEXT:    retq507;508; AVX1-LABEL: umulo_v6i32:509; AVX1:       # %bb.0:510; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3511; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]512; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4513; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]514; AVX1-NEXT:    vpmuludq %xmm2, %xmm5, %xmm2515; AVX1-NEXT:    vpmuludq %xmm3, %xmm4, %xmm5516; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]517; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3],xmm5[4,5],xmm2[6,7]518; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5519; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm2, %xmm2520; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6521; AVX1-NEXT:    vpxor %xmm6, %xmm2, %xmm2522; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]523; AVX1-NEXT:    vpshufd {{.*#+}} xmm8 = xmm0[1,1,3,3]524; AVX1-NEXT:    vpmuludq %xmm7, %xmm8, %xmm7525; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm8526; AVX1-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]527; AVX1-NEXT:    vpblendw {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3],xmm8[4,5],xmm7[6,7]528; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm7, %xmm5529; AVX1-NEXT:    vpxor %xmm6, %xmm5, %xmm5530; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2531; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0532; AVX1-NEXT:    vpmulld %xmm3, %xmm4, %xmm1533; AVX1-NEXT:    vmovq %xmm1, 16(%rdi)534; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)535; AVX1-NEXT:    vmovaps %ymm2, %ymm0536; AVX1-NEXT:    retq537;538; AVX2-LABEL: umulo_v6i32:539; AVX2:       # %bb.0:540; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]541; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]542; AVX2-NEXT:    vpmuludq %ymm2, %ymm3, %ymm2543; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm3544; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]545; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]546; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3547; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm2548; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3549; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2550; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0551; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1552; AVX2-NEXT:    vmovq %xmm1, 16(%rdi)553; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)554; AVX2-NEXT:    vmovdqa %ymm2, %ymm0555; AVX2-NEXT:    retq556;557; AVX512-LABEL: umulo_v6i32:558; AVX512:       # %bb.0:559; AVX512-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2560; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]561; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]562; AVX512-NEXT:    vpmuludq %ymm3, %ymm4, %ymm3563; AVX512-NEXT:    vpmovsxbd {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]564; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4565; AVX512-NEXT:    vptestmd %ymm4, %ymm4, %k1566; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1567; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0568; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}569; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2570; AVX512-NEXT:    vmovq %xmm2, 16(%rdi)571; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)572; AVX512-NEXT:    retq573  %t = call {<6 x i32>, <6 x i1>} @llvm.umul.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1)574  %val = extractvalue {<6 x i32>, <6 x i1>} %t, 0575  %obit = extractvalue {<6 x i32>, <6 x i1>} %t, 1576  %res = sext <6 x i1> %obit to <6 x i32>577  store <6 x i32> %val, ptr %p2578  ret <6 x i32> %res579}580 581define <8 x i32> @umulo_v8i32(<8 x i32> %a0, <8 x i32> %a1, ptr %p2) nounwind {582; SSE2-LABEL: umulo_v8i32:583; SSE2:       # %bb.0:584; SSE2-NEXT:    movdqa %xmm0, %xmm4585; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]586; SSE2-NEXT:    pmuludq %xmm2, %xmm4587; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,3,2,3]588; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]589; SSE2-NEXT:    pmuludq %xmm5, %xmm6590; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3]591; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]592; SSE2-NEXT:    pxor %xmm5, %xmm5593; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0594; SSE2-NEXT:    pcmpeqd %xmm7, %xmm7595; SSE2-NEXT:    pxor %xmm7, %xmm0596; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm1[1,1,3,3]597; SSE2-NEXT:    pmuludq %xmm3, %xmm1598; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]599; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]600; SSE2-NEXT:    pmuludq %xmm8, %xmm3601; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm3[1,3,2,3]602; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm8[0],xmm2[1],xmm8[1]603; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2604; SSE2-NEXT:    pxor %xmm7, %xmm2605; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]606; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]607; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]608; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]609; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]610; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]611; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)612; SSE2-NEXT:    movdqa %xmm4, (%rdi)613; SSE2-NEXT:    movdqa %xmm2, %xmm1614; SSE2-NEXT:    retq615;616; SSSE3-LABEL: umulo_v8i32:617; SSSE3:       # %bb.0:618; SSSE3-NEXT:    movdqa %xmm0, %xmm4619; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]620; SSSE3-NEXT:    pmuludq %xmm2, %xmm4621; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,3,2,3]622; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]623; SSSE3-NEXT:    pmuludq %xmm5, %xmm6624; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,3,2,3]625; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]626; SSSE3-NEXT:    pxor %xmm5, %xmm5627; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm0628; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm7629; SSSE3-NEXT:    pxor %xmm7, %xmm0630; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm1[1,1,3,3]631; SSSE3-NEXT:    pmuludq %xmm3, %xmm1632; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]633; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]634; SSSE3-NEXT:    pmuludq %xmm8, %xmm3635; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm3[1,3,2,3]636; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm8[0],xmm2[1],xmm8[1]637; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2638; SSSE3-NEXT:    pxor %xmm7, %xmm2639; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]640; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]641; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]642; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]643; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]644; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]645; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)646; SSSE3-NEXT:    movdqa %xmm4, (%rdi)647; SSSE3-NEXT:    movdqa %xmm2, %xmm1648; SSSE3-NEXT:    retq649;650; SSE41-LABEL: umulo_v8i32:651; SSE41:       # %bb.0:652; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]653; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]654; SSE41-NEXT:    pmuludq %xmm4, %xmm5655; SSE41-NEXT:    movdqa %xmm0, %xmm4656; SSE41-NEXT:    pmuludq %xmm2, %xmm4657; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]658; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]659; SSE41-NEXT:    pxor %xmm6, %xmm6660; SSE41-NEXT:    pcmpeqd %xmm6, %xmm4661; SSE41-NEXT:    pcmpeqd %xmm7, %xmm7662; SSE41-NEXT:    pxor %xmm7, %xmm4663; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]664; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm1[1,1,3,3]665; SSE41-NEXT:    pmuludq %xmm5, %xmm8666; SSE41-NEXT:    movdqa %xmm1, %xmm5667; SSE41-NEXT:    pmuludq %xmm3, %xmm5668; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]669; SSE41-NEXT:    pblendw {{.*#+}} xmm5 = xmm5[0,1],xmm8[2,3],xmm5[4,5],xmm8[6,7]670; SSE41-NEXT:    pcmpeqd %xmm6, %xmm5671; SSE41-NEXT:    pxor %xmm7, %xmm5672; SSE41-NEXT:    pmulld %xmm2, %xmm0673; SSE41-NEXT:    pmulld %xmm3, %xmm1674; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)675; SSE41-NEXT:    movdqa %xmm0, (%rdi)676; SSE41-NEXT:    movdqa %xmm4, %xmm0677; SSE41-NEXT:    movdqa %xmm5, %xmm1678; SSE41-NEXT:    retq679;680; AVX1-LABEL: umulo_v8i32:681; AVX1:       # %bb.0:682; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3683; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]684; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4685; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]686; AVX1-NEXT:    vpmuludq %xmm2, %xmm5, %xmm2687; AVX1-NEXT:    vpmuludq %xmm3, %xmm4, %xmm5688; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]689; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3],xmm5[4,5],xmm2[6,7]690; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5691; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm2, %xmm2692; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6693; AVX1-NEXT:    vpxor %xmm6, %xmm2, %xmm2694; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]695; AVX1-NEXT:    vpshufd {{.*#+}} xmm8 = xmm0[1,1,3,3]696; AVX1-NEXT:    vpmuludq %xmm7, %xmm8, %xmm7697; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm8698; AVX1-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]699; AVX1-NEXT:    vpblendw {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3],xmm8[4,5],xmm7[6,7]700; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm7, %xmm5701; AVX1-NEXT:    vpxor %xmm6, %xmm5, %xmm5702; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2703; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0704; AVX1-NEXT:    vpmulld %xmm3, %xmm4, %xmm1705; AVX1-NEXT:    vmovdqa %xmm1, 16(%rdi)706; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)707; AVX1-NEXT:    vmovaps %ymm2, %ymm0708; AVX1-NEXT:    retq709;710; AVX2-LABEL: umulo_v8i32:711; AVX2:       # %bb.0:712; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]713; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]714; AVX2-NEXT:    vpmuludq %ymm2, %ymm3, %ymm2715; AVX2-NEXT:    vpmuludq %ymm1, %ymm0, %ymm3716; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]717; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]718; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3719; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm2, %ymm2720; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3721; AVX2-NEXT:    vpxor %ymm3, %ymm2, %ymm2722; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0723; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)724; AVX2-NEXT:    vmovdqa %ymm2, %ymm0725; AVX2-NEXT:    retq726;727; AVX512-LABEL: umulo_v8i32:728; AVX512:       # %bb.0:729; AVX512-NEXT:    vpmuludq %ymm1, %ymm0, %ymm2730; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]731; AVX512-NEXT:    vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]732; AVX512-NEXT:    vpmuludq %ymm3, %ymm4, %ymm3733; AVX512-NEXT:    vpmovsxbd {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]734; AVX512-NEXT:    vpermi2d %ymm3, %ymm2, %ymm4735; AVX512-NEXT:    vptestmd %ymm4, %ymm4, %k1736; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm1737; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0738; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}739; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)740; AVX512-NEXT:    retq741  %t = call {<8 x i32>, <8 x i1>} @llvm.umul.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1)742  %val = extractvalue {<8 x i32>, <8 x i1>} %t, 0743  %obit = extractvalue {<8 x i32>, <8 x i1>} %t, 1744  %res = sext <8 x i1> %obit to <8 x i32>745  store <8 x i32> %val, ptr %p2746  ret <8 x i32> %res747}748 749define <16 x i32> @umulo_v16i32(<16 x i32> %a0, <16 x i32> %a1, ptr %p2) nounwind {750; SSE2-LABEL: umulo_v16i32:751; SSE2:       # %bb.0:752; SSE2-NEXT:    movdqa %xmm0, %xmm9753; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3]754; SSE2-NEXT:    pmuludq %xmm4, %xmm9755; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[1,3,2,3]756; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm4[1,1,3,3]757; SSE2-NEXT:    pmuludq %xmm8, %xmm10758; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm10[1,3,2,3]759; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]760; SSE2-NEXT:    pxor %xmm11, %xmm11761; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0762; SSE2-NEXT:    pcmpeqd %xmm12, %xmm12763; SSE2-NEXT:    pxor %xmm12, %xmm0764; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm1[1,1,3,3]765; SSE2-NEXT:    pmuludq %xmm5, %xmm1766; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,3,2,3]767; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]768; SSE2-NEXT:    pmuludq %xmm8, %xmm5769; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[1,3,2,3]770; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm8[0],xmm4[1],xmm8[1]771; SSE2-NEXT:    pcmpeqd %xmm11, %xmm4772; SSE2-NEXT:    pxor %xmm12, %xmm4773; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm2[1,1,3,3]774; SSE2-NEXT:    pmuludq %xmm6, %xmm2775; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[1,3,2,3]776; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[1,1,3,3]777; SSE2-NEXT:    pmuludq %xmm14, %xmm13778; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm13[1,3,2,3]779; SSE2-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm6[0],xmm8[1],xmm6[1]780; SSE2-NEXT:    pcmpeqd %xmm11, %xmm8781; SSE2-NEXT:    pxor %xmm12, %xmm8782; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm3[1,1,3,3]783; SSE2-NEXT:    pmuludq %xmm7, %xmm3784; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,3,2,3]785; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]786; SSE2-NEXT:    pmuludq %xmm14, %xmm7787; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm7[1,3,2,3]788; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm14[0],xmm6[1],xmm14[1]789; SSE2-NEXT:    pcmpeqd %xmm11, %xmm6790; SSE2-NEXT:    pxor %xmm12, %xmm6791; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[0,2,2,3]792; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[0,2,2,3]793; SSE2-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]794; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]795; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]796; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]797; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]798; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm13[0,2,2,3]799; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]800; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]801; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[0,2,2,3]802; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]803; SSE2-NEXT:    movdqa %xmm3, 48(%rdi)804; SSE2-NEXT:    movdqa %xmm2, 32(%rdi)805; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)806; SSE2-NEXT:    movdqa %xmm9, (%rdi)807; SSE2-NEXT:    movdqa %xmm4, %xmm1808; SSE2-NEXT:    movdqa %xmm8, %xmm2809; SSE2-NEXT:    movdqa %xmm6, %xmm3810; SSE2-NEXT:    retq811;812; SSSE3-LABEL: umulo_v16i32:813; SSSE3:       # %bb.0:814; SSSE3-NEXT:    movdqa %xmm0, %xmm9815; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3]816; SSSE3-NEXT:    pmuludq %xmm4, %xmm9817; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[1,3,2,3]818; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm4[1,1,3,3]819; SSSE3-NEXT:    pmuludq %xmm8, %xmm10820; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm10[1,3,2,3]821; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]822; SSSE3-NEXT:    pxor %xmm11, %xmm11823; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm0824; SSSE3-NEXT:    pcmpeqd %xmm12, %xmm12825; SSSE3-NEXT:    pxor %xmm12, %xmm0826; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm1[1,1,3,3]827; SSSE3-NEXT:    pmuludq %xmm5, %xmm1828; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,3,2,3]829; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]830; SSSE3-NEXT:    pmuludq %xmm8, %xmm5831; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[1,3,2,3]832; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm8[0],xmm4[1],xmm8[1]833; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm4834; SSSE3-NEXT:    pxor %xmm12, %xmm4835; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm2[1,1,3,3]836; SSSE3-NEXT:    pmuludq %xmm6, %xmm2837; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[1,3,2,3]838; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[1,1,3,3]839; SSSE3-NEXT:    pmuludq %xmm14, %xmm13840; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm13[1,3,2,3]841; SSSE3-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm6[0],xmm8[1],xmm6[1]842; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm8843; SSSE3-NEXT:    pxor %xmm12, %xmm8844; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm3[1,1,3,3]845; SSSE3-NEXT:    pmuludq %xmm7, %xmm3846; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[1,3,2,3]847; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]848; SSSE3-NEXT:    pmuludq %xmm14, %xmm7849; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm7[1,3,2,3]850; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm14[0],xmm6[1],xmm14[1]851; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm6852; SSSE3-NEXT:    pxor %xmm12, %xmm6853; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[0,2,2,3]854; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[0,2,2,3]855; SSSE3-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1]856; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]857; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]858; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]859; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]860; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm13[0,2,2,3]861; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]862; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]863; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[0,2,2,3]864; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]865; SSSE3-NEXT:    movdqa %xmm3, 48(%rdi)866; SSSE3-NEXT:    movdqa %xmm2, 32(%rdi)867; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)868; SSSE3-NEXT:    movdqa %xmm9, (%rdi)869; SSSE3-NEXT:    movdqa %xmm4, %xmm1870; SSSE3-NEXT:    movdqa %xmm8, %xmm2871; SSSE3-NEXT:    movdqa %xmm6, %xmm3872; SSSE3-NEXT:    retq873;874; SSE41-LABEL: umulo_v16i32:875; SSE41:       # %bb.0:876; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[1,1,3,3]877; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3]878; SSE41-NEXT:    pmuludq %xmm8, %xmm9879; SSE41-NEXT:    movdqa %xmm0, %xmm8880; SSE41-NEXT:    pmuludq %xmm4, %xmm8881; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]882; SSE41-NEXT:    pblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5],xmm9[6,7]883; SSE41-NEXT:    pxor %xmm12, %xmm12884; SSE41-NEXT:    pcmpeqd %xmm12, %xmm8885; SSE41-NEXT:    pcmpeqd %xmm13, %xmm13886; SSE41-NEXT:    pxor %xmm13, %xmm8887; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm5[1,1,3,3]888; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,3,3]889; SSE41-NEXT:    pmuludq %xmm9, %xmm10890; SSE41-NEXT:    movdqa %xmm1, %xmm9891; SSE41-NEXT:    pmuludq %xmm5, %xmm9892; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]893; SSE41-NEXT:    pblendw {{.*#+}} xmm9 = xmm9[0,1],xmm10[2,3],xmm9[4,5],xmm10[6,7]894; SSE41-NEXT:    pcmpeqd %xmm12, %xmm9895; SSE41-NEXT:    pxor %xmm13, %xmm9896; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm6[1,1,3,3]897; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm2[1,1,3,3]898; SSE41-NEXT:    pmuludq %xmm10, %xmm11899; SSE41-NEXT:    movdqa %xmm2, %xmm10900; SSE41-NEXT:    pmuludq %xmm6, %xmm10901; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3]902; SSE41-NEXT:    pblendw {{.*#+}} xmm10 = xmm10[0,1],xmm11[2,3],xmm10[4,5],xmm11[6,7]903; SSE41-NEXT:    pcmpeqd %xmm12, %xmm10904; SSE41-NEXT:    pxor %xmm13, %xmm10905; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm7[1,1,3,3]906; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm3[1,1,3,3]907; SSE41-NEXT:    pmuludq %xmm11, %xmm14908; SSE41-NEXT:    movdqa %xmm3, %xmm11909; SSE41-NEXT:    pmuludq %xmm7, %xmm11910; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm11[1,1,3,3]911; SSE41-NEXT:    pblendw {{.*#+}} xmm11 = xmm11[0,1],xmm14[2,3],xmm11[4,5],xmm14[6,7]912; SSE41-NEXT:    pcmpeqd %xmm12, %xmm11913; SSE41-NEXT:    pxor %xmm13, %xmm11914; SSE41-NEXT:    pmulld %xmm4, %xmm0915; SSE41-NEXT:    pmulld %xmm5, %xmm1916; SSE41-NEXT:    pmulld %xmm6, %xmm2917; SSE41-NEXT:    pmulld %xmm7, %xmm3918; SSE41-NEXT:    movdqa %xmm3, 48(%rdi)919; SSE41-NEXT:    movdqa %xmm2, 32(%rdi)920; SSE41-NEXT:    movdqa %xmm1, 16(%rdi)921; SSE41-NEXT:    movdqa %xmm0, (%rdi)922; SSE41-NEXT:    movdqa %xmm8, %xmm0923; SSE41-NEXT:    movdqa %xmm9, %xmm1924; SSE41-NEXT:    movdqa %xmm10, %xmm2925; SSE41-NEXT:    movdqa %xmm11, %xmm3926; SSE41-NEXT:    retq927;928; AVX1-LABEL: umulo_v16i32:929; AVX1:       # %bb.0:930; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4931; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]932; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5933; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm5[1,1,3,3]934; AVX1-NEXT:    vpmuludq %xmm6, %xmm7, %xmm6935; AVX1-NEXT:    vpmuludq %xmm4, %xmm5, %xmm7936; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]937; AVX1-NEXT:    vpblendw {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3],xmm7[4,5],xmm6[6,7]938; AVX1-NEXT:    vpxor %xmm7, %xmm7, %xmm7939; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm6, %xmm6940; AVX1-NEXT:    vpshufd {{.*#+}} xmm8 = xmm3[1,1,3,3]941; AVX1-NEXT:    vpshufd {{.*#+}} xmm9 = xmm1[1,1,3,3]942; AVX1-NEXT:    vpmuludq %xmm8, %xmm9, %xmm8943; AVX1-NEXT:    vpmuludq %xmm3, %xmm1, %xmm9944; AVX1-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[1,1,3,3]945; AVX1-NEXT:    vpblendw {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3],xmm9[4,5],xmm8[6,7]946; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm8, %xmm8947; AVX1-NEXT:    vpackssdw %xmm6, %xmm8, %xmm6948; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm8949; AVX1-NEXT:    vpshufd {{.*#+}} xmm9 = xmm8[1,1,3,3]950; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm10951; AVX1-NEXT:    vpshufd {{.*#+}} xmm11 = xmm10[1,1,3,3]952; AVX1-NEXT:    vpmuludq %xmm9, %xmm11, %xmm9953; AVX1-NEXT:    vpmuludq %xmm8, %xmm10, %xmm11954; AVX1-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[1,1,3,3]955; AVX1-NEXT:    vpblendw {{.*#+}} xmm9 = xmm11[0,1],xmm9[2,3],xmm11[4,5],xmm9[6,7]956; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm9, %xmm9957; AVX1-NEXT:    vpshufd {{.*#+}} xmm11 = xmm2[1,1,3,3]958; AVX1-NEXT:    vpshufd {{.*#+}} xmm12 = xmm0[1,1,3,3]959; AVX1-NEXT:    vpmuludq %xmm11, %xmm12, %xmm11960; AVX1-NEXT:    vpmuludq %xmm2, %xmm0, %xmm12961; AVX1-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]962; AVX1-NEXT:    vpblendw {{.*#+}} xmm11 = xmm12[0,1],xmm11[2,3],xmm12[4,5],xmm11[6,7]963; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm11, %xmm7964; AVX1-NEXT:    vpackssdw %xmm9, %xmm7, %xmm7965; AVX1-NEXT:    vpacksswb %xmm6, %xmm7, %xmm7966; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm9, %xmm9967; AVX1-NEXT:    vpxor %xmm7, %xmm9, %xmm7968; AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm2969; AVX1-NEXT:    vpmulld %xmm8, %xmm10, %xmm8970; AVX1-NEXT:    vpmulld %xmm3, %xmm1, %xmm3971; AVX1-NEXT:    vpmulld %xmm4, %xmm5, %xmm4972; AVX1-NEXT:    vpmovsxbd %xmm7, %xmm0973; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm7[1,1,1,1]974; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1975; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0976; AVX1-NEXT:    vpacksswb %xmm6, %xmm6, %xmm1977; AVX1-NEXT:    vpxor %xmm1, %xmm9, %xmm1978; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm5979; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]980; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1981; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm5, %ymm1982; AVX1-NEXT:    vmovdqa %xmm4, 48(%rdi)983; AVX1-NEXT:    vmovdqa %xmm3, 32(%rdi)984; AVX1-NEXT:    vmovdqa %xmm8, 16(%rdi)985; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)986; AVX1-NEXT:    retq987;988; AVX2-LABEL: umulo_v16i32:989; AVX2:       # %bb.0:990; AVX2-NEXT:    vpshufd {{.*#+}} ymm4 = ymm3[1,1,3,3,5,5,7,7]991; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm1[1,1,3,3,5,5,7,7]992; AVX2-NEXT:    vpmuludq %ymm4, %ymm5, %ymm4993; AVX2-NEXT:    vpmuludq %ymm3, %ymm1, %ymm5994; AVX2-NEXT:    vpshufd {{.*#+}} ymm5 = ymm5[1,1,3,3,5,5,7,7]995; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7]996; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5997; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm4, %ymm4998; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6999; AVX2-NEXT:    vpxor %ymm6, %ymm4, %ymm41000; AVX2-NEXT:    vextracti128 $1, %ymm4, %xmm71001; AVX2-NEXT:    vpackssdw %xmm7, %xmm4, %xmm41002; AVX2-NEXT:    vpshufd {{.*#+}} ymm7 = ymm2[1,1,3,3,5,5,7,7]1003; AVX2-NEXT:    vpshufd {{.*#+}} ymm8 = ymm0[1,1,3,3,5,5,7,7]1004; AVX2-NEXT:    vpmuludq %ymm7, %ymm8, %ymm71005; AVX2-NEXT:    vpmuludq %ymm2, %ymm0, %ymm81006; AVX2-NEXT:    vpshufd {{.*#+}} ymm8 = ymm8[1,1,3,3,5,5,7,7]1007; AVX2-NEXT:    vpblendd {{.*#+}} ymm7 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4],ymm7[5],ymm8[6],ymm7[7]1008; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm51009; AVX2-NEXT:    vpmulld %ymm2, %ymm0, %ymm21010; AVX2-NEXT:    vpmulld %ymm3, %ymm1, %ymm31011; AVX2-NEXT:    vpacksswb %xmm4, %xmm4, %xmm01012; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm11013; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm5[0,0,0,0,4,4,4,4,8,8,8,8,12,12,12,12,16,16,16,16,20,20,20,20,24,24,24,24,28,28,28,28]1014; AVX2-NEXT:    vpxor %ymm6, %ymm0, %ymm01015; AVX2-NEXT:    vmovdqa %ymm3, 32(%rdi)1016; AVX2-NEXT:    vmovdqa %ymm2, (%rdi)1017; AVX2-NEXT:    retq1018;1019; AVX512-LABEL: umulo_v16i32:1020; AVX512:       # %bb.0:1021; AVX512-NEXT:    vpmuludq %zmm1, %zmm0, %zmm21022; AVX512-NEXT:    vpshufd {{.*#+}} zmm3 = zmm1[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]1023; AVX512-NEXT:    vpshufd {{.*#+}} zmm4 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]1024; AVX512-NEXT:    vpmuludq %zmm3, %zmm4, %zmm31025; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm4 = [1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31]1026; AVX512-NEXT:    vpermi2d %zmm3, %zmm2, %zmm41027; AVX512-NEXT:    vptestmd %zmm4, %zmm4, %k11028; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm11029; AVX512-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -11030; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdi)1031; AVX512-NEXT:    retq1032  %t = call {<16 x i32>, <16 x i1>} @llvm.umul.with.overflow.v16i32(<16 x i32> %a0, <16 x i32> %a1)1033  %val = extractvalue {<16 x i32>, <16 x i1>} %t, 01034  %obit = extractvalue {<16 x i32>, <16 x i1>} %t, 11035  %res = sext <16 x i1> %obit to <16 x i32>1036  store <16 x i32> %val, ptr %p21037  ret <16 x i32> %res1038}1039 1040define <16 x i32> @umulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, ptr %p2) nounwind {1041; SSE2-LABEL: umulo_v16i8:1042; SSE2:       # %bb.0:1043; SSE2-NEXT:    pxor %xmm2, %xmm21044; SSE2-NEXT:    movdqa %xmm1, %xmm31045; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]1046; SSE2-NEXT:    movdqa %xmm0, %xmm51047; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]1048; SSE2-NEXT:    pmullw %xmm3, %xmm51049; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]1050; SSE2-NEXT:    movdqa %xmm5, %xmm31051; SSE2-NEXT:    pand %xmm4, %xmm31052; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]1053; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]1054; SSE2-NEXT:    pmullw %xmm1, %xmm01055; SSE2-NEXT:    pand %xmm0, %xmm41056; SSE2-NEXT:    packuswb %xmm3, %xmm41057; SSE2-NEXT:    psrlw $8, %xmm51058; SSE2-NEXT:    psrlw $8, %xmm01059; SSE2-NEXT:    packuswb %xmm5, %xmm01060; SSE2-NEXT:    pcmpeqb %xmm0, %xmm21061; SSE2-NEXT:    pcmpeqd %xmm3, %xmm31062; SSE2-NEXT:    pxor %xmm2, %xmm31063; SSE2-NEXT:    movdqa %xmm3, %xmm01064; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]1065; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]1066; SSE2-NEXT:    movdqa %xmm3, %xmm11067; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1068; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]1069; SSE2-NEXT:    pslld $31, %xmm11070; SSE2-NEXT:    psrad $31, %xmm11071; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1072; SSE2-NEXT:    movdqa %xmm3, %xmm21073; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]1074; SSE2-NEXT:    pslld $31, %xmm21075; SSE2-NEXT:    psrad $31, %xmm21076; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]1077; SSE2-NEXT:    pslld $31, %xmm31078; SSE2-NEXT:    psrad $31, %xmm31079; SSE2-NEXT:    movdqa %xmm4, (%rdi)1080; SSE2-NEXT:    retq1081;1082; SSSE3-LABEL: umulo_v16i8:1083; SSSE3:       # %bb.0:1084; SSSE3-NEXT:    pxor %xmm2, %xmm21085; SSSE3-NEXT:    movdqa %xmm1, %xmm31086; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15]1087; SSSE3-NEXT:    movdqa %xmm0, %xmm51088; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]1089; SSSE3-NEXT:    pmullw %xmm3, %xmm51090; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]1091; SSSE3-NEXT:    movdqa %xmm5, %xmm31092; SSSE3-NEXT:    pand %xmm4, %xmm31093; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]1094; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]1095; SSSE3-NEXT:    pmullw %xmm1, %xmm01096; SSSE3-NEXT:    pand %xmm0, %xmm41097; SSSE3-NEXT:    packuswb %xmm3, %xmm41098; SSSE3-NEXT:    psrlw $8, %xmm51099; SSSE3-NEXT:    psrlw $8, %xmm01100; SSSE3-NEXT:    packuswb %xmm5, %xmm01101; SSSE3-NEXT:    pcmpeqb %xmm0, %xmm21102; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm31103; SSSE3-NEXT:    pxor %xmm2, %xmm31104; SSSE3-NEXT:    movdqa %xmm3, %xmm01105; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]1106; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]1107; SSSE3-NEXT:    movdqa %xmm3, %xmm11108; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1109; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]1110; SSSE3-NEXT:    pslld $31, %xmm11111; SSSE3-NEXT:    psrad $31, %xmm11112; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1113; SSSE3-NEXT:    movdqa %xmm3, %xmm21114; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]1115; SSSE3-NEXT:    pslld $31, %xmm21116; SSSE3-NEXT:    psrad $31, %xmm21117; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]1118; SSSE3-NEXT:    pslld $31, %xmm31119; SSSE3-NEXT:    psrad $31, %xmm31120; SSSE3-NEXT:    movdqa %xmm4, (%rdi)1121; SSSE3-NEXT:    retq1122;1123; SSE41-LABEL: umulo_v16i8:1124; SSE41:       # %bb.0:1125; SSE41-NEXT:    pxor %xmm2, %xmm21126; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1127; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]1128; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1129; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]1130; SSE41-NEXT:    pmullw %xmm1, %xmm01131; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]1132; SSE41-NEXT:    movdqa %xmm0, %xmm11133; SSE41-NEXT:    pand %xmm4, %xmm11134; SSE41-NEXT:    pmullw %xmm3, %xmm51135; SSE41-NEXT:    pand %xmm5, %xmm41136; SSE41-NEXT:    packuswb %xmm1, %xmm41137; SSE41-NEXT:    psrlw $8, %xmm01138; SSE41-NEXT:    psrlw $8, %xmm51139; SSE41-NEXT:    packuswb %xmm0, %xmm51140; SSE41-NEXT:    pcmpeqb %xmm2, %xmm51141; SSE41-NEXT:    pcmpeqd %xmm3, %xmm31142; SSE41-NEXT:    pxor %xmm5, %xmm31143; SSE41-NEXT:    pmovsxbd %xmm3, %xmm01144; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]1145; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero1146; SSE41-NEXT:    pslld $31, %xmm11147; SSE41-NEXT:    psrad $31, %xmm11148; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]1149; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero1150; SSE41-NEXT:    pslld $31, %xmm21151; SSE41-NEXT:    psrad $31, %xmm21152; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]1153; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero1154; SSE41-NEXT:    pslld $31, %xmm31155; SSE41-NEXT:    psrad $31, %xmm31156; SSE41-NEXT:    movdqa %xmm4, (%rdi)1157; SSE41-NEXT:    retq1158;1159; AVX1-LABEL: umulo_v16i8:1160; AVX1:       # %bb.0:1161; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm21162; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]1163; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]1164; AVX1-NEXT:    vpmullw %xmm3, %xmm4, %xmm31165; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]1166; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm51167; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1168; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1169; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01170; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm11171; AVX1-NEXT:    vpackuswb %xmm5, %xmm1, %xmm41172; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm11173; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm01174; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm01175; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm01176; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm11177; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm11178; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm01179; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]1180; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm21181; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm01182; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]1183; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm21184; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]1185; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm11186; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm11187; AVX1-NEXT:    vmovdqa %xmm4, (%rdi)1188; AVX1-NEXT:    retq1189;1190; AVX2-LABEL: umulo_v16i8:1191; AVX2:       # %bb.0:1192; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero1193; AVX2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero1194; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm01195; AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm11196; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm21197; AVX2-NEXT:    vpackuswb %xmm2, %xmm1, %xmm21198; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm01199; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm11200; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm01201; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm11202; AVX2-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm01203; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm11204; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm11205; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm01206; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]1207; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm11208; AVX2-NEXT:    vmovdqa %xmm2, (%rdi)1209; AVX2-NEXT:    retq1210;1211; AVX512F-LABEL: umulo_v16i8:1212; AVX512F:       # %bb.0:1213; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero1214; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero1215; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm11216; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm01217; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero1218; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k11219; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -11220; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero1221; AVX512F-NEXT:    vpmovdb %zmm1, (%rdi)1222; AVX512F-NEXT:    retq1223;1224; AVX512BW-LABEL: umulo_v16i8:1225; AVX512BW:       # %bb.0:1226; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero1227; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero1228; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm11229; AVX512BW-NEXT:    vpsrlw $8, %ymm1, %ymm01230; AVX512BW-NEXT:    vptestmw %ymm0, %ymm0, %k11231; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -11232; AVX512BW-NEXT:    vpmovwb %ymm1, (%rdi)1233; AVX512BW-NEXT:    retq1234  %t = call {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)1235  %val = extractvalue {<16 x i8>, <16 x i1>} %t, 01236  %obit = extractvalue {<16 x i8>, <16 x i1>} %t, 11237  %res = sext <16 x i1> %obit to <16 x i32>1238  store <16 x i8> %val, ptr %p21239  ret <16 x i32> %res1240}1241 1242define <32 x i32> @umulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, ptr %p2) nounwind {1243; SSE2-LABEL: umulo_v32i8:1244; SSE2:       # %bb.0:1245; SSE2-NEXT:    movq %rdi, %rax1246; SSE2-NEXT:    pxor %xmm5, %xmm51247; SSE2-NEXT:    movdqa %xmm2, %xmm41248; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15]1249; SSE2-NEXT:    movdqa %xmm0, %xmm61250; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15]1251; SSE2-NEXT:    pmullw %xmm4, %xmm61252; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]1253; SSE2-NEXT:    movdqa %xmm6, %xmm71254; SSE2-NEXT:    pand %xmm4, %xmm71255; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7]1256; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7]1257; SSE2-NEXT:    pmullw %xmm2, %xmm01258; SSE2-NEXT:    movdqa %xmm0, %xmm21259; SSE2-NEXT:    pand %xmm4, %xmm21260; SSE2-NEXT:    packuswb %xmm7, %xmm21261; SSE2-NEXT:    movdqa %xmm3, %xmm71262; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm5[8],xmm7[9],xmm5[9],xmm7[10],xmm5[10],xmm7[11],xmm5[11],xmm7[12],xmm5[12],xmm7[13],xmm5[13],xmm7[14],xmm5[14],xmm7[15],xmm5[15]1263; SSE2-NEXT:    movdqa %xmm1, %xmm81264; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm5[8],xmm8[9],xmm5[9],xmm8[10],xmm5[10],xmm8[11],xmm5[11],xmm8[12],xmm5[12],xmm8[13],xmm5[13],xmm8[14],xmm5[14],xmm8[15],xmm5[15]1265; SSE2-NEXT:    pmullw %xmm7, %xmm81266; SSE2-NEXT:    movdqa %xmm8, %xmm71267; SSE2-NEXT:    pand %xmm4, %xmm71268; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]1269; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3],xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]1270; SSE2-NEXT:    pmullw %xmm3, %xmm11271; SSE2-NEXT:    pand %xmm1, %xmm41272; SSE2-NEXT:    packuswb %xmm7, %xmm41273; SSE2-NEXT:    psrlw $8, %xmm81274; SSE2-NEXT:    psrlw $8, %xmm11275; SSE2-NEXT:    packuswb %xmm8, %xmm11276; SSE2-NEXT:    pcmpeqb %xmm5, %xmm11277; SSE2-NEXT:    pcmpeqd %xmm3, %xmm31278; SSE2-NEXT:    pxor %xmm3, %xmm11279; SSE2-NEXT:    psrlw $8, %xmm61280; SSE2-NEXT:    psrlw $8, %xmm01281; SSE2-NEXT:    packuswb %xmm6, %xmm01282; SSE2-NEXT:    pcmpeqb %xmm5, %xmm01283; SSE2-NEXT:    pxor %xmm3, %xmm01284; SSE2-NEXT:    movdqa %xmm0, %xmm31285; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1286; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]1287; SSE2-NEXT:    pslld $31, %xmm31288; SSE2-NEXT:    psrad $31, %xmm31289; SSE2-NEXT:    movdqa %xmm0, %xmm51290; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1291; SSE2-NEXT:    movdqa %xmm5, %xmm61292; SSE2-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3]1293; SSE2-NEXT:    pslld $31, %xmm61294; SSE2-NEXT:    psrad $31, %xmm61295; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]1296; SSE2-NEXT:    pslld $31, %xmm51297; SSE2-NEXT:    psrad $31, %xmm51298; SSE2-NEXT:    movdqa %xmm1, %xmm71299; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1300; SSE2-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4,4,5,5,6,6,7,7]1301; SSE2-NEXT:    pslld $31, %xmm71302; SSE2-NEXT:    psrad $31, %xmm71303; SSE2-NEXT:    movdqa %xmm1, %xmm81304; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1305; SSE2-NEXT:    movdqa %xmm8, %xmm91306; SSE2-NEXT:    punpcklwd {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3]1307; SSE2-NEXT:    pslld $31, %xmm91308; SSE2-NEXT:    psrad $31, %xmm91309; SSE2-NEXT:    punpckhwd {{.*#+}} xmm8 = xmm8[4,4,5,5,6,6,7,7]1310; SSE2-NEXT:    pslld $31, %xmm81311; SSE2-NEXT:    psrad $31, %xmm81312; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1313; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]1314; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1315; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]1316; SSE2-NEXT:    movdqa %xmm4, 16(%rsi)1317; SSE2-NEXT:    movdqa %xmm2, (%rsi)1318; SSE2-NEXT:    movdqa %xmm1, 64(%rdi)1319; SSE2-NEXT:    movdqa %xmm0, (%rdi)1320; SSE2-NEXT:    movdqa %xmm8, 112(%rdi)1321; SSE2-NEXT:    movdqa %xmm9, 96(%rdi)1322; SSE2-NEXT:    movdqa %xmm7, 80(%rdi)1323; SSE2-NEXT:    movdqa %xmm5, 48(%rdi)1324; SSE2-NEXT:    movdqa %xmm6, 32(%rdi)1325; SSE2-NEXT:    movdqa %xmm3, 16(%rdi)1326; SSE2-NEXT:    retq1327;1328; SSSE3-LABEL: umulo_v32i8:1329; SSSE3:       # %bb.0:1330; SSSE3-NEXT:    movq %rdi, %rax1331; SSSE3-NEXT:    pxor %xmm5, %xmm51332; SSSE3-NEXT:    movdqa %xmm2, %xmm41333; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15]1334; SSSE3-NEXT:    movdqa %xmm0, %xmm61335; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15]1336; SSSE3-NEXT:    pmullw %xmm4, %xmm61337; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]1338; SSSE3-NEXT:    movdqa %xmm6, %xmm71339; SSSE3-NEXT:    pand %xmm4, %xmm71340; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7]1341; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7]1342; SSSE3-NEXT:    pmullw %xmm2, %xmm01343; SSSE3-NEXT:    movdqa %xmm0, %xmm21344; SSSE3-NEXT:    pand %xmm4, %xmm21345; SSSE3-NEXT:    packuswb %xmm7, %xmm21346; SSSE3-NEXT:    movdqa %xmm3, %xmm71347; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm5[8],xmm7[9],xmm5[9],xmm7[10],xmm5[10],xmm7[11],xmm5[11],xmm7[12],xmm5[12],xmm7[13],xmm5[13],xmm7[14],xmm5[14],xmm7[15],xmm5[15]1348; SSSE3-NEXT:    movdqa %xmm1, %xmm81349; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm5[8],xmm8[9],xmm5[9],xmm8[10],xmm5[10],xmm8[11],xmm5[11],xmm8[12],xmm5[12],xmm8[13],xmm5[13],xmm8[14],xmm5[14],xmm8[15],xmm5[15]1350; SSSE3-NEXT:    pmullw %xmm7, %xmm81351; SSSE3-NEXT:    movdqa %xmm8, %xmm71352; SSSE3-NEXT:    pand %xmm4, %xmm71353; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]1354; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3],xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]1355; SSSE3-NEXT:    pmullw %xmm3, %xmm11356; SSSE3-NEXT:    pand %xmm1, %xmm41357; SSSE3-NEXT:    packuswb %xmm7, %xmm41358; SSSE3-NEXT:    psrlw $8, %xmm81359; SSSE3-NEXT:    psrlw $8, %xmm11360; SSSE3-NEXT:    packuswb %xmm8, %xmm11361; SSSE3-NEXT:    pcmpeqb %xmm5, %xmm11362; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm31363; SSSE3-NEXT:    pxor %xmm3, %xmm11364; SSSE3-NEXT:    psrlw $8, %xmm61365; SSSE3-NEXT:    psrlw $8, %xmm01366; SSSE3-NEXT:    packuswb %xmm6, %xmm01367; SSSE3-NEXT:    pcmpeqb %xmm5, %xmm01368; SSSE3-NEXT:    pxor %xmm3, %xmm01369; SSSE3-NEXT:    movdqa %xmm0, %xmm31370; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1371; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]1372; SSSE3-NEXT:    pslld $31, %xmm31373; SSSE3-NEXT:    psrad $31, %xmm31374; SSSE3-NEXT:    movdqa %xmm0, %xmm51375; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1376; SSSE3-NEXT:    movdqa %xmm5, %xmm61377; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3]1378; SSSE3-NEXT:    pslld $31, %xmm61379; SSSE3-NEXT:    psrad $31, %xmm61380; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]1381; SSSE3-NEXT:    pslld $31, %xmm51382; SSSE3-NEXT:    psrad $31, %xmm51383; SSSE3-NEXT:    movdqa %xmm1, %xmm71384; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1385; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4,4,5,5,6,6,7,7]1386; SSSE3-NEXT:    pslld $31, %xmm71387; SSSE3-NEXT:    psrad $31, %xmm71388; SSSE3-NEXT:    movdqa %xmm1, %xmm81389; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1390; SSSE3-NEXT:    movdqa %xmm8, %xmm91391; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3]1392; SSSE3-NEXT:    pslld $31, %xmm91393; SSSE3-NEXT:    psrad $31, %xmm91394; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm8 = xmm8[4,4,5,5,6,6,7,7]1395; SSSE3-NEXT:    pslld $31, %xmm81396; SSSE3-NEXT:    psrad $31, %xmm81397; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1398; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]1399; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1400; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]1401; SSSE3-NEXT:    movdqa %xmm4, 16(%rsi)1402; SSSE3-NEXT:    movdqa %xmm2, (%rsi)1403; SSSE3-NEXT:    movdqa %xmm1, 64(%rdi)1404; SSSE3-NEXT:    movdqa %xmm0, (%rdi)1405; SSSE3-NEXT:    movdqa %xmm8, 112(%rdi)1406; SSSE3-NEXT:    movdqa %xmm9, 96(%rdi)1407; SSSE3-NEXT:    movdqa %xmm7, 80(%rdi)1408; SSSE3-NEXT:    movdqa %xmm5, 48(%rdi)1409; SSSE3-NEXT:    movdqa %xmm6, 32(%rdi)1410; SSSE3-NEXT:    movdqa %xmm3, 16(%rdi)1411; SSSE3-NEXT:    retq1412;1413; SSE41-LABEL: umulo_v32i8:1414; SSE41:       # %bb.0:1415; SSE41-NEXT:    movq %rdi, %rax1416; SSE41-NEXT:    pxor %xmm7, %xmm71417; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero1418; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm7[8],xmm2[9],xmm7[9],xmm2[10],xmm7[10],xmm2[11],xmm7[11],xmm2[12],xmm7[12],xmm2[13],xmm7[13],xmm2[14],xmm7[14],xmm2[15],xmm7[15]1419; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1420; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm7[8],xmm0[9],xmm7[9],xmm0[10],xmm7[10],xmm0[11],xmm7[11],xmm0[12],xmm7[12],xmm0[13],xmm7[13],xmm0[14],xmm7[14],xmm0[15],xmm7[15]1421; SSE41-NEXT:    pmullw %xmm2, %xmm01422; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]1423; SSE41-NEXT:    movdqa %xmm0, %xmm61424; SSE41-NEXT:    pand %xmm2, %xmm61425; SSE41-NEXT:    pmullw %xmm5, %xmm41426; SSE41-NEXT:    movdqa %xmm4, %xmm51427; SSE41-NEXT:    pand %xmm2, %xmm51428; SSE41-NEXT:    packuswb %xmm6, %xmm51429; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm8 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero1430; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm7[8],xmm3[9],xmm7[9],xmm3[10],xmm7[10],xmm3[11],xmm7[11],xmm3[12],xmm7[12],xmm3[13],xmm7[13],xmm3[14],xmm7[14],xmm3[15],xmm7[15]1431; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1432; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm7[8],xmm1[9],xmm7[9],xmm1[10],xmm7[10],xmm1[11],xmm7[11],xmm1[12],xmm7[12],xmm1[13],xmm7[13],xmm1[14],xmm7[14],xmm1[15],xmm7[15]1433; SSE41-NEXT:    pmullw %xmm3, %xmm11434; SSE41-NEXT:    movdqa %xmm1, %xmm31435; SSE41-NEXT:    pand %xmm2, %xmm31436; SSE41-NEXT:    pmullw %xmm8, %xmm61437; SSE41-NEXT:    pand %xmm6, %xmm21438; SSE41-NEXT:    packuswb %xmm3, %xmm21439; SSE41-NEXT:    psrlw $8, %xmm11440; SSE41-NEXT:    psrlw $8, %xmm61441; SSE41-NEXT:    packuswb %xmm1, %xmm61442; SSE41-NEXT:    pcmpeqb %xmm7, %xmm61443; SSE41-NEXT:    pcmpeqd %xmm1, %xmm11444; SSE41-NEXT:    pxor %xmm1, %xmm61445; SSE41-NEXT:    psrlw $8, %xmm01446; SSE41-NEXT:    psrlw $8, %xmm41447; SSE41-NEXT:    packuswb %xmm0, %xmm41448; SSE41-NEXT:    pcmpeqb %xmm7, %xmm41449; SSE41-NEXT:    pxor %xmm1, %xmm41450; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,1,1]1451; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero1452; SSE41-NEXT:    pslld $31, %xmm01453; SSE41-NEXT:    psrad $31, %xmm01454; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[2,3,2,3]1455; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero1456; SSE41-NEXT:    pslld $31, %xmm11457; SSE41-NEXT:    psrad $31, %xmm11458; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[3,3,3,3]1459; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero1460; SSE41-NEXT:    pslld $31, %xmm31461; SSE41-NEXT:    psrad $31, %xmm31462; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,1,1]1463; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero1464; SSE41-NEXT:    pslld $31, %xmm71465; SSE41-NEXT:    psrad $31, %xmm71466; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[2,3,2,3]1467; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero1468; SSE41-NEXT:    pslld $31, %xmm81469; SSE41-NEXT:    psrad $31, %xmm81470; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm6[3,3,3,3]1471; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero1472; SSE41-NEXT:    pslld $31, %xmm91473; SSE41-NEXT:    psrad $31, %xmm91474; SSE41-NEXT:    pmovsxbd %xmm4, %xmm41475; SSE41-NEXT:    pmovsxbd %xmm6, %xmm61476; SSE41-NEXT:    movdqa %xmm2, 16(%rsi)1477; SSE41-NEXT:    movdqa %xmm5, (%rsi)1478; SSE41-NEXT:    movdqa %xmm6, 64(%rdi)1479; SSE41-NEXT:    movdqa %xmm4, (%rdi)1480; SSE41-NEXT:    movdqa %xmm9, 112(%rdi)1481; SSE41-NEXT:    movdqa %xmm8, 96(%rdi)1482; SSE41-NEXT:    movdqa %xmm7, 80(%rdi)1483; SSE41-NEXT:    movdqa %xmm3, 48(%rdi)1484; SSE41-NEXT:    movdqa %xmm1, 32(%rdi)1485; SSE41-NEXT:    movdqa %xmm0, 16(%rdi)1486; SSE41-NEXT:    retq1487;1488; AVX1-LABEL: umulo_v32i8:1489; AVX1:       # %bb.0:1490; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm21491; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]1492; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]1493; AVX1-NEXT:    vpmullw %xmm3, %xmm4, %xmm31494; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]1495; AVX1-NEXT:    vpand %xmm5, %xmm3, %xmm41496; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1497; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1498; AVX1-NEXT:    vpmullw %xmm6, %xmm7, %xmm61499; AVX1-NEXT:    vpand %xmm5, %xmm6, %xmm71500; AVX1-NEXT:    vpackuswb %xmm4, %xmm7, %xmm41501; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm11502; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]1503; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm01504; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]1505; AVX1-NEXT:    vpmullw %xmm7, %xmm8, %xmm71506; AVX1-NEXT:    vpand %xmm5, %xmm7, %xmm81507; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1508; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1509; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm01510; AVX1-NEXT:    vpand %xmm5, %xmm0, %xmm11511; AVX1-NEXT:    vpackuswb %xmm8, %xmm1, %xmm51512; AVX1-NEXT:    vpsrlw $8, %xmm7, %xmm11513; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm01514; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm01515; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm01516; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm11517; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm71518; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm01519; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm31520; AVX1-NEXT:    vpackuswb %xmm0, %xmm3, %xmm01521; AVX1-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm01522; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm11523; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm01524; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]1525; AVX1-NEXT:    vpmovsxbd %xmm2, %xmm21526; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm01527; AVX1-NEXT:    vpmovsxbd %xmm7, %xmm21528; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm7[1,1,1,1]1529; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm31530; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm21531; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]1532; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm31533; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]1534; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm11535; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm11536; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm7[2,3,2,3]1537; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm31538; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm7[3,3,3,3]1539; AVX1-NEXT:    vpmovsxbd %xmm6, %xmm61540; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm3, %ymm31541; AVX1-NEXT:    vmovdqa %xmm5, 16(%rdi)1542; AVX1-NEXT:    vmovdqa %xmm4, (%rdi)1543; AVX1-NEXT:    retq1544;1545; AVX2-LABEL: umulo_v32i8:1546; AVX2:       # %bb.0:1547; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm21548; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]1549; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]1550; AVX2-NEXT:    vpmullw %ymm3, %ymm4, %ymm31551; AVX2-NEXT:    vpbroadcastw {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]1552; AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm51553; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]1554; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]1555; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm01556; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm11557; AVX2-NEXT:    vpackuswb %ymm5, %ymm1, %ymm41558; AVX2-NEXT:    vpsrlw $8, %ymm3, %ymm11559; AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm01560; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm01561; AVX2-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm01562; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm11563; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm11564; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm01565; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm31566; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm21567; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]1568; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm11569; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]1570; AVX2-NEXT:    vpmovsxbd %xmm3, %ymm31571; AVX2-NEXT:    vmovdqa %ymm4, (%rdi)1572; AVX2-NEXT:    retq1573;1574; AVX512F-LABEL: umulo_v32i8:1575; AVX512F:       # %bb.0:1576; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm21577; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero1578; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm31579; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero1580; AVX512F-NEXT:    vpmullw %ymm2, %ymm3, %ymm21581; AVX512F-NEXT:    vpsrlw $8, %ymm2, %ymm31582; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero1583; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k11584; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero1585; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero1586; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm31587; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm01588; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero1589; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k21590; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k2} {z} = -11591; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm1 {%k1} {z} = -11592; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero1593; AVX512F-NEXT:    vpmovdb %zmm2, 16(%rdi)1594; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero1595; AVX512F-NEXT:    vpmovdb %zmm2, (%rdi)1596; AVX512F-NEXT:    retq1597;1598; AVX512BW-LABEL: umulo_v32i8:1599; AVX512BW:       # %bb.0:1600; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero1601; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero1602; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm21603; AVX512BW-NEXT:    vpsrlw $8, %zmm2, %zmm01604; AVX512BW-NEXT:    vptestmw %zmm0, %zmm0, %k11605; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -11606; AVX512BW-NEXT:    kshiftrd $16, %k1, %k11607; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm1 {%k1} {z} = -11608; AVX512BW-NEXT:    vpmovwb %zmm2, (%rdi)1609; AVX512BW-NEXT:    retq1610  %t = call {<32 x i8>, <32 x i1>} @llvm.umul.with.overflow.v32i8(<32 x i8> %a0, <32 x i8> %a1)1611  %val = extractvalue {<32 x i8>, <32 x i1>} %t, 01612  %obit = extractvalue {<32 x i8>, <32 x i1>} %t, 11613  %res = sext <32 x i1> %obit to <32 x i32>1614  store <32 x i8> %val, ptr %p21615  ret <32 x i32> %res1616}1617 1618define <64 x i32> @umulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, ptr %p2) nounwind {1619; SSE2-LABEL: umulo_v64i8:1620; SSE2:       # %bb.0:1621; SSE2-NEXT:    movq %rdi, %rax1622; SSE2-NEXT:    pxor %xmm10, %xmm101623; SSE2-NEXT:    movdqa %xmm4, %xmm81624; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm10[8],xmm8[9],xmm10[9],xmm8[10],xmm10[10],xmm8[11],xmm10[11],xmm8[12],xmm10[12],xmm8[13],xmm10[13],xmm8[14],xmm10[14],xmm8[15],xmm10[15]1625; SSE2-NEXT:    movdqa %xmm0, %xmm111626; SSE2-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm10[8],xmm11[9],xmm10[9],xmm11[10],xmm10[10],xmm11[11],xmm10[11],xmm11[12],xmm10[12],xmm11[13],xmm10[13],xmm11[14],xmm10[14],xmm11[15],xmm10[15]1627; SSE2-NEXT:    pmullw %xmm8, %xmm111628; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]1629; SSE2-NEXT:    movdqa %xmm11, %xmm91630; SSE2-NEXT:    pand %xmm8, %xmm91631; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm10[0],xmm4[1],xmm10[1],xmm4[2],xmm10[2],xmm4[3],xmm10[3],xmm4[4],xmm10[4],xmm4[5],xmm10[5],xmm4[6],xmm10[6],xmm4[7],xmm10[7]1632; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]1633; SSE2-NEXT:    pmullw %xmm4, %xmm01634; SSE2-NEXT:    movdqa %xmm0, %xmm41635; SSE2-NEXT:    pand %xmm8, %xmm41636; SSE2-NEXT:    packuswb %xmm9, %xmm41637; SSE2-NEXT:    movdqa %xmm5, %xmm91638; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm10[8],xmm9[9],xmm10[9],xmm9[10],xmm10[10],xmm9[11],xmm10[11],xmm9[12],xmm10[12],xmm9[13],xmm10[13],xmm9[14],xmm10[14],xmm9[15],xmm10[15]1639; SSE2-NEXT:    movdqa %xmm1, %xmm121640; SSE2-NEXT:    punpckhbw {{.*#+}} xmm12 = xmm12[8],xmm10[8],xmm12[9],xmm10[9],xmm12[10],xmm10[10],xmm12[11],xmm10[11],xmm12[12],xmm10[12],xmm12[13],xmm10[13],xmm12[14],xmm10[14],xmm12[15],xmm10[15]1641; SSE2-NEXT:    pmullw %xmm9, %xmm121642; SSE2-NEXT:    movdqa %xmm12, %xmm91643; SSE2-NEXT:    pand %xmm8, %xmm91644; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm10[0],xmm5[1],xmm10[1],xmm5[2],xmm10[2],xmm5[3],xmm10[3],xmm5[4],xmm10[4],xmm5[5],xmm10[5],xmm5[6],xmm10[6],xmm5[7],xmm10[7]1645; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1],xmm1[2],xmm10[2],xmm1[3],xmm10[3],xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]1646; SSE2-NEXT:    pmullw %xmm5, %xmm11647; SSE2-NEXT:    movdqa %xmm1, %xmm51648; SSE2-NEXT:    pand %xmm8, %xmm51649; SSE2-NEXT:    packuswb %xmm9, %xmm51650; SSE2-NEXT:    movdqa %xmm6, %xmm91651; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm10[8],xmm9[9],xmm10[9],xmm9[10],xmm10[10],xmm9[11],xmm10[11],xmm9[12],xmm10[12],xmm9[13],xmm10[13],xmm9[14],xmm10[14],xmm9[15],xmm10[15]1652; SSE2-NEXT:    movdqa %xmm2, %xmm131653; SSE2-NEXT:    punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm10[8],xmm13[9],xmm10[9],xmm13[10],xmm10[10],xmm13[11],xmm10[11],xmm13[12],xmm10[12],xmm13[13],xmm10[13],xmm13[14],xmm10[14],xmm13[15],xmm10[15]1654; SSE2-NEXT:    pmullw %xmm9, %xmm131655; SSE2-NEXT:    movdqa %xmm13, %xmm141656; SSE2-NEXT:    pand %xmm8, %xmm141657; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm10[0],xmm6[1],xmm10[1],xmm6[2],xmm10[2],xmm6[3],xmm10[3],xmm6[4],xmm10[4],xmm6[5],xmm10[5],xmm6[6],xmm10[6],xmm6[7],xmm10[7]1658; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3],xmm2[4],xmm10[4],xmm2[5],xmm10[5],xmm2[6],xmm10[6],xmm2[7],xmm10[7]1659; SSE2-NEXT:    pmullw %xmm6, %xmm21660; SSE2-NEXT:    movdqa %xmm2, %xmm91661; SSE2-NEXT:    pand %xmm8, %xmm91662; SSE2-NEXT:    packuswb %xmm14, %xmm91663; SSE2-NEXT:    movdqa %xmm7, %xmm61664; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm10[8],xmm6[9],xmm10[9],xmm6[10],xmm10[10],xmm6[11],xmm10[11],xmm6[12],xmm10[12],xmm6[13],xmm10[13],xmm6[14],xmm10[14],xmm6[15],xmm10[15]1665; SSE2-NEXT:    movdqa %xmm3, %xmm141666; SSE2-NEXT:    punpckhbw {{.*#+}} xmm14 = xmm14[8],xmm10[8],xmm14[9],xmm10[9],xmm14[10],xmm10[10],xmm14[11],xmm10[11],xmm14[12],xmm10[12],xmm14[13],xmm10[13],xmm14[14],xmm10[14],xmm14[15],xmm10[15]1667; SSE2-NEXT:    pmullw %xmm6, %xmm141668; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm10[0],xmm7[1],xmm10[1],xmm7[2],xmm10[2],xmm7[3],xmm10[3],xmm7[4],xmm10[4],xmm7[5],xmm10[5],xmm7[6],xmm10[6],xmm7[7],xmm10[7]1669; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3],xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]1670; SSE2-NEXT:    pmullw %xmm7, %xmm31671; SSE2-NEXT:    movdqa %xmm14, %xmm61672; SSE2-NEXT:    pand %xmm8, %xmm61673; SSE2-NEXT:    pand %xmm3, %xmm81674; SSE2-NEXT:    packuswb %xmm6, %xmm81675; SSE2-NEXT:    psrlw $8, %xmm141676; SSE2-NEXT:    psrlw $8, %xmm31677; SSE2-NEXT:    packuswb %xmm14, %xmm31678; SSE2-NEXT:    psrlw $8, %xmm131679; SSE2-NEXT:    psrlw $8, %xmm21680; SSE2-NEXT:    packuswb %xmm13, %xmm21681; SSE2-NEXT:    psrlw $8, %xmm121682; SSE2-NEXT:    psrlw $8, %xmm11683; SSE2-NEXT:    packuswb %xmm12, %xmm11684; SSE2-NEXT:    psrlw $8, %xmm111685; SSE2-NEXT:    psrlw $8, %xmm01686; SSE2-NEXT:    packuswb %xmm11, %xmm01687; SSE2-NEXT:    pcmpeqb %xmm10, %xmm31688; SSE2-NEXT:    pcmpeqb %xmm10, %xmm21689; SSE2-NEXT:    pcmpeqb %xmm10, %xmm11690; SSE2-NEXT:    pcmpeqb %xmm10, %xmm01691; SSE2-NEXT:    pcmpeqd %xmm6, %xmm61692; SSE2-NEXT:    pxor %xmm6, %xmm31693; SSE2-NEXT:    pxor %xmm6, %xmm21694; SSE2-NEXT:    pxor %xmm6, %xmm11695; SSE2-NEXT:    pxor %xmm6, %xmm01696; SSE2-NEXT:    movdqa %xmm0, %xmm61697; SSE2-NEXT:    movdqa %xmm8, 48(%rsi)1698; SSE2-NEXT:    movdqa %xmm1, %xmm71699; SSE2-NEXT:    movdqa %xmm9, 32(%rsi)1700; SSE2-NEXT:    movdqa %xmm2, %xmm81701; SSE2-NEXT:    movdqa %xmm5, 16(%rsi)1702; SSE2-NEXT:    movdqa %xmm3, %xmm51703; SSE2-NEXT:    movdqa %xmm4, (%rsi)1704; SSE2-NEXT:    movdqa %xmm3, %xmm41705; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1706; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]1707; SSE2-NEXT:    movdqa %xmm3, 192(%rdi)1708; SSE2-NEXT:    movdqa %xmm2, %xmm31709; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1710; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]1711; SSE2-NEXT:    movdqa %xmm2, 128(%rdi)1712; SSE2-NEXT:    movdqa %xmm1, %xmm21713; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1714; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]1715; SSE2-NEXT:    movdqa %xmm1, 64(%rdi)1716; SSE2-NEXT:    movdqa %xmm0, %xmm11717; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1718; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1719; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]1720; SSE2-NEXT:    movdqa %xmm0, (%rdi)1721; SSE2-NEXT:    movdqa %xmm4, %xmm01722; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]1723; SSE2-NEXT:    pslld $31, %xmm41724; SSE2-NEXT:    psrad $31, %xmm41725; SSE2-NEXT:    movdqa %xmm4, 224(%rdi)1726; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]1727; SSE2-NEXT:    pslld $31, %xmm01728; SSE2-NEXT:    psrad $31, %xmm01729; SSE2-NEXT:    movdqa %xmm0, 240(%rdi)1730; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1731; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1732; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]1733; SSE2-NEXT:    pslld $31, %xmm51734; SSE2-NEXT:    psrad $31, %xmm51735; SSE2-NEXT:    movdqa %xmm5, 208(%rdi)1736; SSE2-NEXT:    movdqa %xmm3, %xmm01737; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]1738; SSE2-NEXT:    pslld $31, %xmm31739; SSE2-NEXT:    psrad $31, %xmm31740; SSE2-NEXT:    movdqa %xmm3, 160(%rdi)1741; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]1742; SSE2-NEXT:    pslld $31, %xmm01743; SSE2-NEXT:    psrad $31, %xmm01744; SSE2-NEXT:    movdqa %xmm0, 176(%rdi)1745; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1746; SSE2-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1747; SSE2-NEXT:    punpckhwd {{.*#+}} xmm8 = xmm8[4,4,5,5,6,6,7,7]1748; SSE2-NEXT:    pslld $31, %xmm81749; SSE2-NEXT:    psrad $31, %xmm81750; SSE2-NEXT:    movdqa %xmm8, 144(%rdi)1751; SSE2-NEXT:    movdqa %xmm2, %xmm01752; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]1753; SSE2-NEXT:    pslld $31, %xmm21754; SSE2-NEXT:    psrad $31, %xmm21755; SSE2-NEXT:    movdqa %xmm2, 96(%rdi)1756; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]1757; SSE2-NEXT:    pslld $31, %xmm01758; SSE2-NEXT:    psrad $31, %xmm01759; SSE2-NEXT:    movdqa %xmm0, 112(%rdi)1760; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1761; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1762; SSE2-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4,4,5,5,6,6,7,7]1763; SSE2-NEXT:    pslld $31, %xmm71764; SSE2-NEXT:    psrad $31, %xmm71765; SSE2-NEXT:    movdqa %xmm7, 80(%rdi)1766; SSE2-NEXT:    movdqa %xmm1, %xmm01767; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]1768; SSE2-NEXT:    pslld $31, %xmm11769; SSE2-NEXT:    psrad $31, %xmm11770; SSE2-NEXT:    movdqa %xmm1, 32(%rdi)1771; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]1772; SSE2-NEXT:    pslld $31, %xmm01773; SSE2-NEXT:    psrad $31, %xmm01774; SSE2-NEXT:    movdqa %xmm0, 48(%rdi)1775; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1776; SSE2-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4,4,5,5,6,6,7,7]1777; SSE2-NEXT:    pslld $31, %xmm61778; SSE2-NEXT:    psrad $31, %xmm61779; SSE2-NEXT:    movdqa %xmm6, 16(%rdi)1780; SSE2-NEXT:    retq1781;1782; SSSE3-LABEL: umulo_v64i8:1783; SSSE3:       # %bb.0:1784; SSSE3-NEXT:    movq %rdi, %rax1785; SSSE3-NEXT:    pxor %xmm10, %xmm101786; SSSE3-NEXT:    movdqa %xmm4, %xmm81787; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm10[8],xmm8[9],xmm10[9],xmm8[10],xmm10[10],xmm8[11],xmm10[11],xmm8[12],xmm10[12],xmm8[13],xmm10[13],xmm8[14],xmm10[14],xmm8[15],xmm10[15]1788; SSSE3-NEXT:    movdqa %xmm0, %xmm111789; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm10[8],xmm11[9],xmm10[9],xmm11[10],xmm10[10],xmm11[11],xmm10[11],xmm11[12],xmm10[12],xmm11[13],xmm10[13],xmm11[14],xmm10[14],xmm11[15],xmm10[15]1790; SSSE3-NEXT:    pmullw %xmm8, %xmm111791; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]1792; SSSE3-NEXT:    movdqa %xmm11, %xmm91793; SSSE3-NEXT:    pand %xmm8, %xmm91794; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm10[0],xmm4[1],xmm10[1],xmm4[2],xmm10[2],xmm4[3],xmm10[3],xmm4[4],xmm10[4],xmm4[5],xmm10[5],xmm4[6],xmm10[6],xmm4[7],xmm10[7]1795; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]1796; SSSE3-NEXT:    pmullw %xmm4, %xmm01797; SSSE3-NEXT:    movdqa %xmm0, %xmm41798; SSSE3-NEXT:    pand %xmm8, %xmm41799; SSSE3-NEXT:    packuswb %xmm9, %xmm41800; SSSE3-NEXT:    movdqa %xmm5, %xmm91801; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm10[8],xmm9[9],xmm10[9],xmm9[10],xmm10[10],xmm9[11],xmm10[11],xmm9[12],xmm10[12],xmm9[13],xmm10[13],xmm9[14],xmm10[14],xmm9[15],xmm10[15]1802; SSSE3-NEXT:    movdqa %xmm1, %xmm121803; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm12 = xmm12[8],xmm10[8],xmm12[9],xmm10[9],xmm12[10],xmm10[10],xmm12[11],xmm10[11],xmm12[12],xmm10[12],xmm12[13],xmm10[13],xmm12[14],xmm10[14],xmm12[15],xmm10[15]1804; SSSE3-NEXT:    pmullw %xmm9, %xmm121805; SSSE3-NEXT:    movdqa %xmm12, %xmm91806; SSSE3-NEXT:    pand %xmm8, %xmm91807; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm10[0],xmm5[1],xmm10[1],xmm5[2],xmm10[2],xmm5[3],xmm10[3],xmm5[4],xmm10[4],xmm5[5],xmm10[5],xmm5[6],xmm10[6],xmm5[7],xmm10[7]1808; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1],xmm1[2],xmm10[2],xmm1[3],xmm10[3],xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]1809; SSSE3-NEXT:    pmullw %xmm5, %xmm11810; SSSE3-NEXT:    movdqa %xmm1, %xmm51811; SSSE3-NEXT:    pand %xmm8, %xmm51812; SSSE3-NEXT:    packuswb %xmm9, %xmm51813; SSSE3-NEXT:    movdqa %xmm6, %xmm91814; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm10[8],xmm9[9],xmm10[9],xmm9[10],xmm10[10],xmm9[11],xmm10[11],xmm9[12],xmm10[12],xmm9[13],xmm10[13],xmm9[14],xmm10[14],xmm9[15],xmm10[15]1815; SSSE3-NEXT:    movdqa %xmm2, %xmm131816; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm10[8],xmm13[9],xmm10[9],xmm13[10],xmm10[10],xmm13[11],xmm10[11],xmm13[12],xmm10[12],xmm13[13],xmm10[13],xmm13[14],xmm10[14],xmm13[15],xmm10[15]1817; SSSE3-NEXT:    pmullw %xmm9, %xmm131818; SSSE3-NEXT:    movdqa %xmm13, %xmm141819; SSSE3-NEXT:    pand %xmm8, %xmm141820; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm10[0],xmm6[1],xmm10[1],xmm6[2],xmm10[2],xmm6[3],xmm10[3],xmm6[4],xmm10[4],xmm6[5],xmm10[5],xmm6[6],xmm10[6],xmm6[7],xmm10[7]1821; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3],xmm2[4],xmm10[4],xmm2[5],xmm10[5],xmm2[6],xmm10[6],xmm2[7],xmm10[7]1822; SSSE3-NEXT:    pmullw %xmm6, %xmm21823; SSSE3-NEXT:    movdqa %xmm2, %xmm91824; SSSE3-NEXT:    pand %xmm8, %xmm91825; SSSE3-NEXT:    packuswb %xmm14, %xmm91826; SSSE3-NEXT:    movdqa %xmm7, %xmm61827; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm10[8],xmm6[9],xmm10[9],xmm6[10],xmm10[10],xmm6[11],xmm10[11],xmm6[12],xmm10[12],xmm6[13],xmm10[13],xmm6[14],xmm10[14],xmm6[15],xmm10[15]1828; SSSE3-NEXT:    movdqa %xmm3, %xmm141829; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm14 = xmm14[8],xmm10[8],xmm14[9],xmm10[9],xmm14[10],xmm10[10],xmm14[11],xmm10[11],xmm14[12],xmm10[12],xmm14[13],xmm10[13],xmm14[14],xmm10[14],xmm14[15],xmm10[15]1830; SSSE3-NEXT:    pmullw %xmm6, %xmm141831; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm10[0],xmm7[1],xmm10[1],xmm7[2],xmm10[2],xmm7[3],xmm10[3],xmm7[4],xmm10[4],xmm7[5],xmm10[5],xmm7[6],xmm10[6],xmm7[7],xmm10[7]1832; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3],xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]1833; SSSE3-NEXT:    pmullw %xmm7, %xmm31834; SSSE3-NEXT:    movdqa %xmm14, %xmm61835; SSSE3-NEXT:    pand %xmm8, %xmm61836; SSSE3-NEXT:    pand %xmm3, %xmm81837; SSSE3-NEXT:    packuswb %xmm6, %xmm81838; SSSE3-NEXT:    psrlw $8, %xmm141839; SSSE3-NEXT:    psrlw $8, %xmm31840; SSSE3-NEXT:    packuswb %xmm14, %xmm31841; SSSE3-NEXT:    psrlw $8, %xmm131842; SSSE3-NEXT:    psrlw $8, %xmm21843; SSSE3-NEXT:    packuswb %xmm13, %xmm21844; SSSE3-NEXT:    psrlw $8, %xmm121845; SSSE3-NEXT:    psrlw $8, %xmm11846; SSSE3-NEXT:    packuswb %xmm12, %xmm11847; SSSE3-NEXT:    psrlw $8, %xmm111848; SSSE3-NEXT:    psrlw $8, %xmm01849; SSSE3-NEXT:    packuswb %xmm11, %xmm01850; SSSE3-NEXT:    pcmpeqb %xmm10, %xmm31851; SSSE3-NEXT:    pcmpeqb %xmm10, %xmm21852; SSSE3-NEXT:    pcmpeqb %xmm10, %xmm11853; SSSE3-NEXT:    pcmpeqb %xmm10, %xmm01854; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm61855; SSSE3-NEXT:    pxor %xmm6, %xmm31856; SSSE3-NEXT:    pxor %xmm6, %xmm21857; SSSE3-NEXT:    pxor %xmm6, %xmm11858; SSSE3-NEXT:    pxor %xmm6, %xmm01859; SSSE3-NEXT:    movdqa %xmm0, %xmm61860; SSSE3-NEXT:    movdqa %xmm8, 48(%rsi)1861; SSSE3-NEXT:    movdqa %xmm1, %xmm71862; SSSE3-NEXT:    movdqa %xmm9, 32(%rsi)1863; SSSE3-NEXT:    movdqa %xmm2, %xmm81864; SSSE3-NEXT:    movdqa %xmm5, 16(%rsi)1865; SSSE3-NEXT:    movdqa %xmm3, %xmm51866; SSSE3-NEXT:    movdqa %xmm4, (%rsi)1867; SSSE3-NEXT:    movdqa %xmm3, %xmm41868; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1869; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]1870; SSSE3-NEXT:    movdqa %xmm3, 192(%rdi)1871; SSSE3-NEXT:    movdqa %xmm2, %xmm31872; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1873; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]1874; SSSE3-NEXT:    movdqa %xmm2, 128(%rdi)1875; SSSE3-NEXT:    movdqa %xmm1, %xmm21876; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1877; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]1878; SSSE3-NEXT:    movdqa %xmm1, 64(%rdi)1879; SSSE3-NEXT:    movdqa %xmm0, %xmm11880; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1881; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1882; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]1883; SSSE3-NEXT:    movdqa %xmm0, (%rdi)1884; SSSE3-NEXT:    movdqa %xmm4, %xmm01885; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]1886; SSSE3-NEXT:    pslld $31, %xmm41887; SSSE3-NEXT:    psrad $31, %xmm41888; SSSE3-NEXT:    movdqa %xmm4, 224(%rdi)1889; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]1890; SSSE3-NEXT:    pslld $31, %xmm01891; SSSE3-NEXT:    psrad $31, %xmm01892; SSSE3-NEXT:    movdqa %xmm0, 240(%rdi)1893; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1894; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1895; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]1896; SSSE3-NEXT:    pslld $31, %xmm51897; SSSE3-NEXT:    psrad $31, %xmm51898; SSSE3-NEXT:    movdqa %xmm5, 208(%rdi)1899; SSSE3-NEXT:    movdqa %xmm3, %xmm01900; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]1901; SSSE3-NEXT:    pslld $31, %xmm31902; SSSE3-NEXT:    psrad $31, %xmm31903; SSSE3-NEXT:    movdqa %xmm3, 160(%rdi)1904; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]1905; SSSE3-NEXT:    pslld $31, %xmm01906; SSSE3-NEXT:    psrad $31, %xmm01907; SSSE3-NEXT:    movdqa %xmm0, 176(%rdi)1908; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1909; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1910; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm8 = xmm8[4,4,5,5,6,6,7,7]1911; SSSE3-NEXT:    pslld $31, %xmm81912; SSSE3-NEXT:    psrad $31, %xmm81913; SSSE3-NEXT:    movdqa %xmm8, 144(%rdi)1914; SSSE3-NEXT:    movdqa %xmm2, %xmm01915; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]1916; SSSE3-NEXT:    pslld $31, %xmm21917; SSSE3-NEXT:    psrad $31, %xmm21918; SSSE3-NEXT:    movdqa %xmm2, 96(%rdi)1919; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]1920; SSSE3-NEXT:    pslld $31, %xmm01921; SSSE3-NEXT:    psrad $31, %xmm01922; SSSE3-NEXT:    movdqa %xmm0, 112(%rdi)1923; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1924; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1925; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4,4,5,5,6,6,7,7]1926; SSSE3-NEXT:    pslld $31, %xmm71927; SSSE3-NEXT:    psrad $31, %xmm71928; SSSE3-NEXT:    movdqa %xmm7, 80(%rdi)1929; SSSE3-NEXT:    movdqa %xmm1, %xmm01930; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]1931; SSSE3-NEXT:    pslld $31, %xmm11932; SSSE3-NEXT:    psrad $31, %xmm11933; SSSE3-NEXT:    movdqa %xmm1, 32(%rdi)1934; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]1935; SSSE3-NEXT:    pslld $31, %xmm01936; SSSE3-NEXT:    psrad $31, %xmm01937; SSSE3-NEXT:    movdqa %xmm0, 48(%rdi)1938; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1939; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4,4,5,5,6,6,7,7]1940; SSSE3-NEXT:    pslld $31, %xmm61941; SSSE3-NEXT:    psrad $31, %xmm61942; SSSE3-NEXT:    movdqa %xmm6, 16(%rdi)1943; SSSE3-NEXT:    retq1944;1945; SSE41-LABEL: umulo_v64i8:1946; SSE41:       # %bb.0:1947; SSE41-NEXT:    movq %rdi, %rax1948; SSE41-NEXT:    pxor %xmm13, %xmm131949; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero1950; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm13[8],xmm4[9],xmm13[9],xmm4[10],xmm13[10],xmm4[11],xmm13[11],xmm4[12],xmm13[12],xmm4[13],xmm13[13],xmm4[14],xmm13[14],xmm4[15],xmm13[15]1951; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm8 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero1952; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm13[8],xmm0[9],xmm13[9],xmm0[10],xmm13[10],xmm0[11],xmm13[11],xmm0[12],xmm13[12],xmm0[13],xmm13[13],xmm0[14],xmm13[14],xmm0[15],xmm13[15]1953; SSE41-NEXT:    pmullw %xmm4, %xmm01954; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]1955; SSE41-NEXT:    movdqa %xmm0, %xmm41956; SSE41-NEXT:    pand %xmm9, %xmm41957; SSE41-NEXT:    pmullw %xmm10, %xmm81958; SSE41-NEXT:    movdqa %xmm8, %xmm101959; SSE41-NEXT:    pand %xmm9, %xmm101960; SSE41-NEXT:    packuswb %xmm4, %xmm101961; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm11 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero1962; SSE41-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm13[8],xmm5[9],xmm13[9],xmm5[10],xmm13[10],xmm5[11],xmm13[11],xmm5[12],xmm13[12],xmm5[13],xmm13[13],xmm5[14],xmm13[14],xmm5[15],xmm13[15]1963; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1964; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm13[8],xmm1[9],xmm13[9],xmm1[10],xmm13[10],xmm1[11],xmm13[11],xmm1[12],xmm13[12],xmm1[13],xmm13[13],xmm1[14],xmm13[14],xmm1[15],xmm13[15]1965; SSE41-NEXT:    pmullw %xmm5, %xmm11966; SSE41-NEXT:    movdqa %xmm1, %xmm51967; SSE41-NEXT:    pand %xmm9, %xmm51968; SSE41-NEXT:    pmullw %xmm11, %xmm41969; SSE41-NEXT:    movdqa %xmm4, %xmm111970; SSE41-NEXT:    pand %xmm9, %xmm111971; SSE41-NEXT:    packuswb %xmm5, %xmm111972; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm12 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero1973; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm13[8],xmm6[9],xmm13[9],xmm6[10],xmm13[10],xmm6[11],xmm13[11],xmm6[12],xmm13[12],xmm6[13],xmm13[13],xmm6[14],xmm13[14],xmm6[15],xmm13[15]1974; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero1975; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm13[8],xmm2[9],xmm13[9],xmm2[10],xmm13[10],xmm2[11],xmm13[11],xmm2[12],xmm13[12],xmm2[13],xmm13[13],xmm2[14],xmm13[14],xmm2[15],xmm13[15]1976; SSE41-NEXT:    pmullw %xmm6, %xmm21977; SSE41-NEXT:    movdqa %xmm2, %xmm61978; SSE41-NEXT:    pand %xmm9, %xmm61979; SSE41-NEXT:    pmullw %xmm12, %xmm51980; SSE41-NEXT:    movdqa %xmm5, %xmm121981; SSE41-NEXT:    pand %xmm9, %xmm121982; SSE41-NEXT:    packuswb %xmm6, %xmm121983; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm14 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero1984; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm13[8],xmm7[9],xmm13[9],xmm7[10],xmm13[10],xmm7[11],xmm13[11],xmm7[12],xmm13[12],xmm7[13],xmm13[13],xmm7[14],xmm13[14],xmm7[15],xmm13[15]1985; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm6 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero1986; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm13[8],xmm3[9],xmm13[9],xmm3[10],xmm13[10],xmm3[11],xmm13[11],xmm3[12],xmm13[12],xmm3[13],xmm13[13],xmm3[14],xmm13[14],xmm3[15],xmm13[15]1987; SSE41-NEXT:    pmullw %xmm7, %xmm31988; SSE41-NEXT:    pmullw %xmm14, %xmm61989; SSE41-NEXT:    movdqa %xmm3, %xmm71990; SSE41-NEXT:    pand %xmm9, %xmm71991; SSE41-NEXT:    pand %xmm6, %xmm91992; SSE41-NEXT:    packuswb %xmm7, %xmm91993; SSE41-NEXT:    psrlw $8, %xmm31994; SSE41-NEXT:    psrlw $8, %xmm61995; SSE41-NEXT:    packuswb %xmm3, %xmm61996; SSE41-NEXT:    psrlw $8, %xmm21997; SSE41-NEXT:    psrlw $8, %xmm51998; SSE41-NEXT:    packuswb %xmm2, %xmm51999; SSE41-NEXT:    psrlw $8, %xmm12000; SSE41-NEXT:    psrlw $8, %xmm42001; SSE41-NEXT:    packuswb %xmm1, %xmm42002; SSE41-NEXT:    psrlw $8, %xmm02003; SSE41-NEXT:    psrlw $8, %xmm82004; SSE41-NEXT:    packuswb %xmm0, %xmm82005; SSE41-NEXT:    pcmpeqb %xmm13, %xmm62006; SSE41-NEXT:    pcmpeqb %xmm13, %xmm52007; SSE41-NEXT:    pcmpeqb %xmm13, %xmm42008; SSE41-NEXT:    pcmpeqb %xmm13, %xmm82009; SSE41-NEXT:    pcmpeqd %xmm0, %xmm02010; SSE41-NEXT:    pxor %xmm0, %xmm62011; SSE41-NEXT:    pxor %xmm0, %xmm52012; SSE41-NEXT:    pxor %xmm0, %xmm42013; SSE41-NEXT:    pxor %xmm0, %xmm82014; SSE41-NEXT:    movdqa %xmm9, 48(%rsi)2015; SSE41-NEXT:    movdqa %xmm12, 32(%rsi)2016; SSE41-NEXT:    movdqa %xmm11, 16(%rsi)2017; SSE41-NEXT:    movdqa %xmm10, (%rsi)2018; SSE41-NEXT:    pmovsxbd %xmm6, %xmm02019; SSE41-NEXT:    movdqa %xmm0, 192(%rdi)2020; SSE41-NEXT:    pmovsxbd %xmm5, %xmm02021; SSE41-NEXT:    movdqa %xmm0, 128(%rdi)2022; SSE41-NEXT:    pmovsxbd %xmm4, %xmm02023; SSE41-NEXT:    movdqa %xmm0, 64(%rdi)2024; SSE41-NEXT:    pmovsxbd %xmm8, %xmm02025; SSE41-NEXT:    movdqa %xmm0, (%rdi)2026; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[2,3,2,3]2027; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2028; SSE41-NEXT:    pslld $31, %xmm02029; SSE41-NEXT:    psrad $31, %xmm02030; SSE41-NEXT:    movdqa %xmm0, 224(%rdi)2031; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[3,3,3,3]2032; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2033; SSE41-NEXT:    pslld $31, %xmm02034; SSE41-NEXT:    psrad $31, %xmm02035; SSE41-NEXT:    movdqa %xmm0, 240(%rdi)2036; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,1,1]2037; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2038; SSE41-NEXT:    pslld $31, %xmm02039; SSE41-NEXT:    psrad $31, %xmm02040; SSE41-NEXT:    movdqa %xmm0, 208(%rdi)2041; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]2042; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2043; SSE41-NEXT:    pslld $31, %xmm02044; SSE41-NEXT:    psrad $31, %xmm02045; SSE41-NEXT:    movdqa %xmm0, 160(%rdi)2046; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[3,3,3,3]2047; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2048; SSE41-NEXT:    pslld $31, %xmm02049; SSE41-NEXT:    psrad $31, %xmm02050; SSE41-NEXT:    movdqa %xmm0, 176(%rdi)2051; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,1,1]2052; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2053; SSE41-NEXT:    pslld $31, %xmm02054; SSE41-NEXT:    psrad $31, %xmm02055; SSE41-NEXT:    movdqa %xmm0, 144(%rdi)2056; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]2057; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2058; SSE41-NEXT:    pslld $31, %xmm02059; SSE41-NEXT:    psrad $31, %xmm02060; SSE41-NEXT:    movdqa %xmm0, 96(%rdi)2061; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[3,3,3,3]2062; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2063; SSE41-NEXT:    pslld $31, %xmm02064; SSE41-NEXT:    psrad $31, %xmm02065; SSE41-NEXT:    movdqa %xmm0, 112(%rdi)2066; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,1,1]2067; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2068; SSE41-NEXT:    pslld $31, %xmm02069; SSE41-NEXT:    psrad $31, %xmm02070; SSE41-NEXT:    movdqa %xmm0, 80(%rdi)2071; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[2,3,2,3]2072; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2073; SSE41-NEXT:    pslld $31, %xmm02074; SSE41-NEXT:    psrad $31, %xmm02075; SSE41-NEXT:    movdqa %xmm0, 32(%rdi)2076; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[3,3,3,3]2077; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2078; SSE41-NEXT:    pslld $31, %xmm02079; SSE41-NEXT:    psrad $31, %xmm02080; SSE41-NEXT:    movdqa %xmm0, 48(%rdi)2081; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm8[1,1,1,1]2082; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2083; SSE41-NEXT:    pslld $31, %xmm02084; SSE41-NEXT:    psrad $31, %xmm02085; SSE41-NEXT:    movdqa %xmm0, 16(%rdi)2086; SSE41-NEXT:    retq2087;2088; AVX1-LABEL: umulo_v64i8:2089; AVX1:       # %bb.0:2090; AVX1-NEXT:    movq %rdi, %rax2091; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm52092; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15]2093; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm0[8],xmm5[8],xmm0[9],xmm5[9],xmm0[10],xmm5[10],xmm0[11],xmm5[11],xmm0[12],xmm5[12],xmm0[13],xmm5[13],xmm0[14],xmm5[14],xmm0[15],xmm5[15]2094; AVX1-NEXT:    vpmullw %xmm4, %xmm6, %xmm62095; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255]2096; AVX1-NEXT:    vpand %xmm7, %xmm6, %xmm42097; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm8 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero2098; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm9 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero2099; AVX1-NEXT:    vpmullw %xmm8, %xmm9, %xmm82100; AVX1-NEXT:    vpand %xmm7, %xmm8, %xmm92101; AVX1-NEXT:    vpackuswb %xmm4, %xmm9, %xmm42102; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm22103; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15]2104; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm02105; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm0[8],xmm5[8],xmm0[9],xmm5[9],xmm0[10],xmm5[10],xmm0[11],xmm5[11],xmm0[12],xmm5[12],xmm0[13],xmm5[13],xmm0[14],xmm5[14],xmm0[15],xmm5[15]2106; AVX1-NEXT:    vpmullw %xmm9, %xmm10, %xmm92107; AVX1-NEXT:    vpand %xmm7, %xmm9, %xmm112108; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero2109; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero2110; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm102111; AVX1-NEXT:    vpand %xmm7, %xmm10, %xmm02112; AVX1-NEXT:    vpackuswb %xmm11, %xmm0, %xmm02113; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]2114; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15]2115; AVX1-NEXT:    vpmullw %xmm2, %xmm11, %xmm112116; AVX1-NEXT:    vpand %xmm7, %xmm11, %xmm22117; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm12 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero2118; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm13 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero2119; AVX1-NEXT:    vpmullw %xmm12, %xmm13, %xmm122120; AVX1-NEXT:    vpand %xmm7, %xmm12, %xmm132121; AVX1-NEXT:    vpackuswb %xmm2, %xmm13, %xmm22122; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm32123; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm13 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]2124; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm12125; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15]2126; AVX1-NEXT:    vpmullw %xmm13, %xmm14, %xmm132127; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero2128; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero2129; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm12130; AVX1-NEXT:    vpand %xmm7, %xmm13, %xmm32131; AVX1-NEXT:    vpand %xmm7, %xmm1, %xmm72132; AVX1-NEXT:    vpackuswb %xmm3, %xmm7, %xmm72133; AVX1-NEXT:    vpsrlw $8, %xmm13, %xmm32134; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm12135; AVX1-NEXT:    vpackuswb %xmm3, %xmm1, %xmm12136; AVX1-NEXT:    vpsrlw $8, %xmm11, %xmm32137; AVX1-NEXT:    vpsrlw $8, %xmm12, %xmm112138; AVX1-NEXT:    vpackuswb %xmm3, %xmm11, %xmm32139; AVX1-NEXT:    vpsrlw $8, %xmm9, %xmm92140; AVX1-NEXT:    vpsrlw $8, %xmm10, %xmm102141; AVX1-NEXT:    vpackuswb %xmm9, %xmm10, %xmm92142; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm62143; AVX1-NEXT:    vpsrlw $8, %xmm8, %xmm82144; AVX1-NEXT:    vpackuswb %xmm6, %xmm8, %xmm62145; AVX1-NEXT:    vpcmpeqb %xmm5, %xmm1, %xmm12146; AVX1-NEXT:    vpcmpeqb %xmm5, %xmm3, %xmm32147; AVX1-NEXT:    vpcmpeqb %xmm5, %xmm9, %xmm82148; AVX1-NEXT:    vpcmpeqb %xmm5, %xmm6, %xmm92149; AVX1-NEXT:    vpcmpeqd %xmm10, %xmm10, %xmm102150; AVX1-NEXT:    vpxor %xmm1, %xmm10, %xmm62151; AVX1-NEXT:    vpxor %xmm3, %xmm10, %xmm52152; AVX1-NEXT:    vpxor %xmm10, %xmm8, %xmm32153; AVX1-NEXT:    vpxor %xmm10, %xmm9, %xmm12154; AVX1-NEXT:    vmovdqa %xmm7, 48(%rsi)2155; AVX1-NEXT:    vmovdqa %xmm2, 32(%rsi)2156; AVX1-NEXT:    vmovdqa %xmm0, 16(%rsi)2157; AVX1-NEXT:    vmovdqa %xmm4, (%rsi)2158; AVX1-NEXT:    vpmovsxbd %xmm6, %xmm02159; AVX1-NEXT:    vmovdqa %xmm0, 192(%rdi)2160; AVX1-NEXT:    vpmovsxbd %xmm5, %xmm02161; AVX1-NEXT:    vmovdqa %xmm0, 128(%rdi)2162; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm02163; AVX1-NEXT:    vmovdqa %xmm0, 64(%rdi)2164; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm02165; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)2166; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm6[2,3,2,3]2167; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm02168; AVX1-NEXT:    vmovdqa %xmm0, 224(%rdi)2169; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm6[3,3,3,3]2170; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm02171; AVX1-NEXT:    vmovdqa %xmm0, 240(%rdi)2172; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm6[1,1,1,1]2173; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm02174; AVX1-NEXT:    vmovdqa %xmm0, 208(%rdi)2175; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]2176; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm02177; AVX1-NEXT:    vmovdqa %xmm0, 160(%rdi)2178; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm5[3,3,3,3]2179; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm02180; AVX1-NEXT:    vmovdqa %xmm0, 176(%rdi)2181; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm5[1,1,1,1]2182; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm02183; AVX1-NEXT:    vmovdqa %xmm0, 144(%rdi)2184; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]2185; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm02186; AVX1-NEXT:    vmovdqa %xmm0, 96(%rdi)2187; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm3[3,3,3,3]2188; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm02189; AVX1-NEXT:    vmovdqa %xmm0, 112(%rdi)2190; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]2191; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm02192; AVX1-NEXT:    vmovdqa %xmm0, 80(%rdi)2193; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]2194; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm02195; AVX1-NEXT:    vmovdqa %xmm0, 32(%rdi)2196; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]2197; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm02198; AVX1-NEXT:    vmovdqa %xmm0, 48(%rdi)2199; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]2200; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm02201; AVX1-NEXT:    vmovdqa %xmm0, 16(%rdi)2202; AVX1-NEXT:    vzeroupper2203; AVX1-NEXT:    retq2204;2205; AVX2-LABEL: umulo_v64i8:2206; AVX2:       # %bb.0:2207; AVX2-NEXT:    movq %rdi, %rax2208; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm42209; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15],ymm2[24],ymm4[24],ymm2[25],ymm4[25],ymm2[26],ymm4[26],ymm2[27],ymm4[27],ymm2[28],ymm4[28],ymm2[29],ymm4[29],ymm2[30],ymm4[30],ymm2[31],ymm4[31]2210; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm6 = ymm0[8],ymm4[8],ymm0[9],ymm4[9],ymm0[10],ymm4[10],ymm0[11],ymm4[11],ymm0[12],ymm4[12],ymm0[13],ymm4[13],ymm0[14],ymm4[14],ymm0[15],ymm4[15],ymm0[24],ymm4[24],ymm0[25],ymm4[25],ymm0[26],ymm4[26],ymm0[27],ymm4[27],ymm0[28],ymm4[28],ymm0[29],ymm4[29],ymm0[30],ymm4[30],ymm0[31],ymm4[31]2211; AVX2-NEXT:    vpmullw %ymm5, %ymm6, %ymm52212; AVX2-NEXT:    vpbroadcastw {{.*#+}} ymm6 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]2213; AVX2-NEXT:    vpand %ymm6, %ymm5, %ymm72214; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[16],ymm4[16],ymm2[17],ymm4[17],ymm2[18],ymm4[18],ymm2[19],ymm4[19],ymm2[20],ymm4[20],ymm2[21],ymm4[21],ymm2[22],ymm4[22],ymm2[23],ymm4[23]2215; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm4[0],ymm0[1],ymm4[1],ymm0[2],ymm4[2],ymm0[3],ymm4[3],ymm0[4],ymm4[4],ymm0[5],ymm4[5],ymm0[6],ymm4[6],ymm0[7],ymm4[7],ymm0[16],ymm4[16],ymm0[17],ymm4[17],ymm0[18],ymm4[18],ymm0[19],ymm4[19],ymm0[20],ymm4[20],ymm0[21],ymm4[21],ymm0[22],ymm4[22],ymm0[23],ymm4[23]2216; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm22217; AVX2-NEXT:    vpand %ymm6, %ymm2, %ymm02218; AVX2-NEXT:    vpackuswb %ymm7, %ymm0, %ymm02219; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm7 = ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15],ymm3[24],ymm4[24],ymm3[25],ymm4[25],ymm3[26],ymm4[26],ymm3[27],ymm4[27],ymm3[28],ymm4[28],ymm3[29],ymm4[29],ymm3[30],ymm4[30],ymm3[31],ymm4[31]2220; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm8 = ymm1[8],ymm4[8],ymm1[9],ymm4[9],ymm1[10],ymm4[10],ymm1[11],ymm4[11],ymm1[12],ymm4[12],ymm1[13],ymm4[13],ymm1[14],ymm4[14],ymm1[15],ymm4[15],ymm1[24],ymm4[24],ymm1[25],ymm4[25],ymm1[26],ymm4[26],ymm1[27],ymm4[27],ymm1[28],ymm4[28],ymm1[29],ymm4[29],ymm1[30],ymm4[30],ymm1[31],ymm4[31]2221; AVX2-NEXT:    vpmullw %ymm7, %ymm8, %ymm72222; AVX2-NEXT:    vpand %ymm6, %ymm7, %ymm82223; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[4],ymm4[4],ymm3[5],ymm4[5],ymm3[6],ymm4[6],ymm3[7],ymm4[7],ymm3[16],ymm4[16],ymm3[17],ymm4[17],ymm3[18],ymm4[18],ymm3[19],ymm4[19],ymm3[20],ymm4[20],ymm3[21],ymm4[21],ymm3[22],ymm4[22],ymm3[23],ymm4[23]2224; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[1],ymm4[1],ymm1[2],ymm4[2],ymm1[3],ymm4[3],ymm1[4],ymm4[4],ymm1[5],ymm4[5],ymm1[6],ymm4[6],ymm1[7],ymm4[7],ymm1[16],ymm4[16],ymm1[17],ymm4[17],ymm1[18],ymm4[18],ymm1[19],ymm4[19],ymm1[20],ymm4[20],ymm1[21],ymm4[21],ymm1[22],ymm4[22],ymm1[23],ymm4[23]2225; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm12226; AVX2-NEXT:    vpand %ymm6, %ymm1, %ymm32227; AVX2-NEXT:    vpackuswb %ymm8, %ymm3, %ymm32228; AVX2-NEXT:    vpsrlw $8, %ymm7, %ymm62229; AVX2-NEXT:    vpsrlw $8, %ymm1, %ymm12230; AVX2-NEXT:    vpackuswb %ymm6, %ymm1, %ymm12231; AVX2-NEXT:    vpcmpeqb %ymm4, %ymm1, %ymm12232; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm62233; AVX2-NEXT:    vpxor %ymm6, %ymm1, %ymm12234; AVX2-NEXT:    vpsrlw $8, %ymm5, %ymm52235; AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm22236; AVX2-NEXT:    vpackuswb %ymm5, %ymm2, %ymm22237; AVX2-NEXT:    vpcmpeqb %ymm4, %ymm2, %ymm22238; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm22239; AVX2-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]2240; AVX2-NEXT:    vpmovsxbd %xmm4, %ymm42241; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm52242; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]2243; AVX2-NEXT:    vpmovsxbd %xmm6, %ymm62244; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm1[2,3,2,3]2245; AVX2-NEXT:    vpmovsxbd %xmm7, %ymm72246; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm82247; AVX2-NEXT:    vpshufd {{.*#+}} xmm9 = xmm8[2,3,2,3]2248; AVX2-NEXT:    vpmovsxbd %xmm9, %ymm92249; AVX2-NEXT:    vpmovsxbd %xmm2, %ymm22250; AVX2-NEXT:    vpmovsxbd %xmm5, %ymm52251; AVX2-NEXT:    vpmovsxbd %xmm1, %ymm12252; AVX2-NEXT:    vpmovsxbd %xmm8, %ymm82253; AVX2-NEXT:    vmovdqa %ymm3, 32(%rsi)2254; AVX2-NEXT:    vmovdqa %ymm0, (%rsi)2255; AVX2-NEXT:    vmovdqa %ymm8, 192(%rdi)2256; AVX2-NEXT:    vmovdqa %ymm1, 128(%rdi)2257; AVX2-NEXT:    vmovdqa %ymm5, 64(%rdi)2258; AVX2-NEXT:    vmovdqa %ymm2, (%rdi)2259; AVX2-NEXT:    vmovdqa %ymm9, 224(%rdi)2260; AVX2-NEXT:    vmovdqa %ymm7, 160(%rdi)2261; AVX2-NEXT:    vmovdqa %ymm6, 96(%rdi)2262; AVX2-NEXT:    vmovdqa %ymm4, 32(%rdi)2263; AVX2-NEXT:    vzeroupper2264; AVX2-NEXT:    retq2265;2266; AVX512F-LABEL: umulo_v64i8:2267; AVX512F:       # %bb.0:2268; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm22269; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm32270; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero2271; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm52272; AVX512F-NEXT:    vextracti128 $1, %ymm5, %xmm42273; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero2274; AVX512F-NEXT:    vpmullw %ymm3, %ymm4, %ymm42275; AVX512F-NEXT:    vpsrlw $8, %ymm4, %ymm32276; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero2277; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k12278; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero2279; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero2280; AVX512F-NEXT:    vpmullw %ymm2, %ymm3, %ymm52281; AVX512F-NEXT:    vpsrlw $8, %ymm5, %ymm22282; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero2283; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k22284; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm22285; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero2286; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm32287; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero2288; AVX512F-NEXT:    vpmullw %ymm2, %ymm3, %ymm62289; AVX512F-NEXT:    vpsrlw $8, %ymm6, %ymm22290; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero2291; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k32292; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero2293; AVX512F-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero2294; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm72295; AVX512F-NEXT:    vpsrlw $8, %ymm7, %ymm02296; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero2297; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k42298; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm0 {%k4} {z} = -12299; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm1 {%k3} {z} = -12300; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm2 {%k2} {z} = -12301; AVX512F-NEXT:    vpternlogd {{.*#+}} zmm3 {%k1} {z} = -12302; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero2303; AVX512F-NEXT:    vpmovdb %zmm4, 48(%rdi)2304; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm5[0],zero,ymm5[1],zero,ymm5[2],zero,ymm5[3],zero,ymm5[4],zero,ymm5[5],zero,ymm5[6],zero,ymm5[7],zero,ymm5[8],zero,ymm5[9],zero,ymm5[10],zero,ymm5[11],zero,ymm5[12],zero,ymm5[13],zero,ymm5[14],zero,ymm5[15],zero2305; AVX512F-NEXT:    vpmovdb %zmm4, 32(%rdi)2306; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm6[0],zero,ymm6[1],zero,ymm6[2],zero,ymm6[3],zero,ymm6[4],zero,ymm6[5],zero,ymm6[6],zero,ymm6[7],zero,ymm6[8],zero,ymm6[9],zero,ymm6[10],zero,ymm6[11],zero,ymm6[12],zero,ymm6[13],zero,ymm6[14],zero,ymm6[15],zero2307; AVX512F-NEXT:    vpmovdb %zmm4, 16(%rdi)2308; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm4 = ymm7[0],zero,ymm7[1],zero,ymm7[2],zero,ymm7[3],zero,ymm7[4],zero,ymm7[5],zero,ymm7[6],zero,ymm7[7],zero,ymm7[8],zero,ymm7[9],zero,ymm7[10],zero,ymm7[11],zero,ymm7[12],zero,ymm7[13],zero,ymm7[14],zero,ymm7[15],zero2309; AVX512F-NEXT:    vpmovdb %zmm4, (%rdi)2310; AVX512F-NEXT:    retq2311;2312; AVX512BW-LABEL: umulo_v64i8:2313; AVX512BW:       # %bb.0:2314; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm22315; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]2316; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm4 = zmm0[8],zmm2[8],zmm0[9],zmm2[9],zmm0[10],zmm2[10],zmm0[11],zmm2[11],zmm0[12],zmm2[12],zmm0[13],zmm2[13],zmm0[14],zmm2[14],zmm0[15],zmm2[15],zmm0[24],zmm2[24],zmm0[25],zmm2[25],zmm0[26],zmm2[26],zmm0[27],zmm2[27],zmm0[28],zmm2[28],zmm0[29],zmm2[29],zmm0[30],zmm2[30],zmm0[31],zmm2[31],zmm0[40],zmm2[40],zmm0[41],zmm2[41],zmm0[42],zmm2[42],zmm0[43],zmm2[43],zmm0[44],zmm2[44],zmm0[45],zmm2[45],zmm0[46],zmm2[46],zmm0[47],zmm2[47],zmm0[56],zmm2[56],zmm0[57],zmm2[57],zmm0[58],zmm2[58],zmm0[59],zmm2[59],zmm0[60],zmm2[60],zmm0[61],zmm2[61],zmm0[62],zmm2[62],zmm0[63],zmm2[63]2317; AVX512BW-NEXT:    vpmullw %zmm3, %zmm4, %zmm32318; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} zmm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]2319; AVX512BW-NEXT:    vpandq %zmm4, %zmm3, %zmm52320; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm2[0],zmm1[1],zmm2[1],zmm1[2],zmm2[2],zmm1[3],zmm2[3],zmm1[4],zmm2[4],zmm1[5],zmm2[5],zmm1[6],zmm2[6],zmm1[7],zmm2[7],zmm1[16],zmm2[16],zmm1[17],zmm2[17],zmm1[18],zmm2[18],zmm1[19],zmm2[19],zmm1[20],zmm2[20],zmm1[21],zmm2[21],zmm1[22],zmm2[22],zmm1[23],zmm2[23],zmm1[32],zmm2[32],zmm1[33],zmm2[33],zmm1[34],zmm2[34],zmm1[35],zmm2[35],zmm1[36],zmm2[36],zmm1[37],zmm2[37],zmm1[38],zmm2[38],zmm1[39],zmm2[39],zmm1[48],zmm2[48],zmm1[49],zmm2[49],zmm1[50],zmm2[50],zmm1[51],zmm2[51],zmm1[52],zmm2[52],zmm1[53],zmm2[53],zmm1[54],zmm2[54],zmm1[55],zmm2[55]2321; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0],zmm2[0],zmm0[1],zmm2[1],zmm0[2],zmm2[2],zmm0[3],zmm2[3],zmm0[4],zmm2[4],zmm0[5],zmm2[5],zmm0[6],zmm2[6],zmm0[7],zmm2[7],zmm0[16],zmm2[16],zmm0[17],zmm2[17],zmm0[18],zmm2[18],zmm0[19],zmm2[19],zmm0[20],zmm2[20],zmm0[21],zmm2[21],zmm0[22],zmm2[22],zmm0[23],zmm2[23],zmm0[32],zmm2[32],zmm0[33],zmm2[33],zmm0[34],zmm2[34],zmm0[35],zmm2[35],zmm0[36],zmm2[36],zmm0[37],zmm2[37],zmm0[38],zmm2[38],zmm0[39],zmm2[39],zmm0[48],zmm2[48],zmm0[49],zmm2[49],zmm0[50],zmm2[50],zmm0[51],zmm2[51],zmm0[52],zmm2[52],zmm0[53],zmm2[53],zmm0[54],zmm2[54],zmm0[55],zmm2[55]2322; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm02323; AVX512BW-NEXT:    vpandq %zmm4, %zmm0, %zmm12324; AVX512BW-NEXT:    vpackuswb %zmm5, %zmm1, %zmm42325; AVX512BW-NEXT:    vpsrlw $8, %zmm3, %zmm12326; AVX512BW-NEXT:    vpsrlw $8, %zmm0, %zmm02327; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm02328; AVX512BW-NEXT:    vptestmb %zmm0, %zmm0, %k12329; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -12330; AVX512BW-NEXT:    kshiftrq $16, %k1, %k22331; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm1 {%k2} {z} = -12332; AVX512BW-NEXT:    kshiftrq $32, %k1, %k22333; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm2 {%k2} {z} = -12334; AVX512BW-NEXT:    kshiftrq $48, %k1, %k12335; AVX512BW-NEXT:    vpternlogd {{.*#+}} zmm3 {%k1} {z} = -12336; AVX512BW-NEXT:    vmovdqa64 %zmm4, (%rdi)2337; AVX512BW-NEXT:    retq2338  %t = call {<64 x i8>, <64 x i1>} @llvm.umul.with.overflow.v64i8(<64 x i8> %a0, <64 x i8> %a1)2339  %val = extractvalue {<64 x i8>, <64 x i1>} %t, 02340  %obit = extractvalue {<64 x i8>, <64 x i1>} %t, 12341  %res = sext <64 x i1> %obit to <64 x i32>2342  store <64 x i8> %val, ptr %p22343  ret <64 x i32> %res2344}2345 2346define <8 x i32> @umulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, ptr %p2) nounwind {2347; SSE2-LABEL: umulo_v8i16:2348; SSE2:       # %bb.0:2349; SSE2-NEXT:    movdqa %xmm0, %xmm22350; SSE2-NEXT:    pmullw %xmm1, %xmm22351; SSE2-NEXT:    pmulhuw %xmm0, %xmm12352; SSE2-NEXT:    pxor %xmm0, %xmm02353; SSE2-NEXT:    pcmpeqw %xmm0, %xmm12354; SSE2-NEXT:    pcmpeqd %xmm0, %xmm02355; SSE2-NEXT:    pxor %xmm0, %xmm12356; SSE2-NEXT:    movdqa %xmm1, %xmm02357; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]2358; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]2359; SSE2-NEXT:    pslld $31, %xmm12360; SSE2-NEXT:    psrad $31, %xmm12361; SSE2-NEXT:    movdqa %xmm2, (%rdi)2362; SSE2-NEXT:    retq2363;2364; SSSE3-LABEL: umulo_v8i16:2365; SSSE3:       # %bb.0:2366; SSSE3-NEXT:    movdqa %xmm0, %xmm22367; SSSE3-NEXT:    pmullw %xmm1, %xmm22368; SSSE3-NEXT:    pmulhuw %xmm0, %xmm12369; SSSE3-NEXT:    pxor %xmm0, %xmm02370; SSSE3-NEXT:    pcmpeqw %xmm0, %xmm12371; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm02372; SSSE3-NEXT:    pxor %xmm0, %xmm12373; SSSE3-NEXT:    movdqa %xmm1, %xmm02374; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]2375; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]2376; SSSE3-NEXT:    pslld $31, %xmm12377; SSSE3-NEXT:    psrad $31, %xmm12378; SSSE3-NEXT:    movdqa %xmm2, (%rdi)2379; SSSE3-NEXT:    retq2380;2381; SSE41-LABEL: umulo_v8i16:2382; SSE41:       # %bb.0:2383; SSE41-NEXT:    movdqa %xmm0, %xmm22384; SSE41-NEXT:    pmullw %xmm1, %xmm22385; SSE41-NEXT:    pmulhuw %xmm0, %xmm12386; SSE41-NEXT:    pxor %xmm0, %xmm02387; SSE41-NEXT:    pcmpeqw %xmm0, %xmm12388; SSE41-NEXT:    pcmpeqd %xmm0, %xmm02389; SSE41-NEXT:    pxor %xmm0, %xmm12390; SSE41-NEXT:    pmovsxwd %xmm1, %xmm02391; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]2392; SSE41-NEXT:    pslld $31, %xmm12393; SSE41-NEXT:    psrad $31, %xmm12394; SSE41-NEXT:    movdqa %xmm2, (%rdi)2395; SSE41-NEXT:    retq2396;2397; AVX1-LABEL: umulo_v8i16:2398; AVX1:       # %bb.0:2399; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm22400; AVX1-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm02401; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm12402; AVX1-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm02403; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm12404; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm02405; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm12406; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]2407; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm02408; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)2409; AVX1-NEXT:    retq2410;2411; AVX2-LABEL: umulo_v8i16:2412; AVX2:       # %bb.0:2413; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm22414; AVX2-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm02415; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm12416; AVX2-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm02417; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm12418; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm02419; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm02420; AVX2-NEXT:    vmovdqa %xmm2, (%rdi)2421; AVX2-NEXT:    retq2422;2423; AVX512F-LABEL: umulo_v8i16:2424; AVX512F:       # %bb.0:2425; AVX512F-NEXT:    vpmullw %xmm1, %xmm0, %xmm22426; AVX512F-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm02427; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm12428; AVX512F-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm02429; AVX512F-NEXT:    vpternlogq {{.*#+}} xmm0 = ~xmm02430; AVX512F-NEXT:    vpmovsxwd %xmm0, %ymm02431; AVX512F-NEXT:    vptestmd %ymm0, %ymm0, %k12432; AVX512F-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm02433; AVX512F-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}2434; AVX512F-NEXT:    vmovdqa %xmm2, (%rdi)2435; AVX512F-NEXT:    retq2436;2437; AVX512BW-LABEL: umulo_v8i16:2438; AVX512BW:       # %bb.0:2439; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm22440; AVX512BW-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm02441; AVX512BW-NEXT:    vptestmw %xmm0, %xmm0, %k12442; AVX512BW-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm02443; AVX512BW-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}2444; AVX512BW-NEXT:    vmovdqa %xmm2, (%rdi)2445; AVX512BW-NEXT:    retq2446  %t = call {<8 x i16>, <8 x i1>} @llvm.umul.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)2447  %val = extractvalue {<8 x i16>, <8 x i1>} %t, 02448  %obit = extractvalue {<8 x i16>, <8 x i1>} %t, 12449  %res = sext <8 x i1> %obit to <8 x i32>2450  store <8 x i16> %val, ptr %p22451  ret <8 x i32> %res2452}2453 2454define <2 x i32> @umulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, ptr %p2) nounwind {2455; SSE2-LABEL: umulo_v2i64:2456; SSE2:       # %bb.0:2457; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]2458; SSE2-NEXT:    movq %xmm2, %rcx2459; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]2460; SSE2-NEXT:    movq %xmm2, %rsi2461; SSE2-NEXT:    movq %xmm0, %rax2462; SSE2-NEXT:    movq %xmm1, %rdx2463; SSE2-NEXT:    xorl %r8d, %r8d2464; SSE2-NEXT:    mulq %rdx2465; SSE2-NEXT:    movq $-1, %r92466; SSE2-NEXT:    movl $0, %r10d2467; SSE2-NEXT:    cmovoq %r9, %r102468; SSE2-NEXT:    movq %rax, %xmm12469; SSE2-NEXT:    movq %rcx, %rax2470; SSE2-NEXT:    mulq %rsi2471; SSE2-NEXT:    movq %rax, %xmm02472; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]2473; SSE2-NEXT:    movq %r10, %xmm02474; SSE2-NEXT:    cmovoq %r9, %r82475; SSE2-NEXT:    movq %r8, %xmm22476; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]2477; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]2478; SSE2-NEXT:    movdqa %xmm1, (%rdi)2479; SSE2-NEXT:    retq2480;2481; SSSE3-LABEL: umulo_v2i64:2482; SSSE3:       # %bb.0:2483; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]2484; SSSE3-NEXT:    movq %xmm2, %rcx2485; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]2486; SSSE3-NEXT:    movq %xmm2, %rsi2487; SSSE3-NEXT:    movq %xmm0, %rax2488; SSSE3-NEXT:    movq %xmm1, %rdx2489; SSSE3-NEXT:    xorl %r8d, %r8d2490; SSSE3-NEXT:    mulq %rdx2491; SSSE3-NEXT:    movq $-1, %r92492; SSSE3-NEXT:    movl $0, %r10d2493; SSSE3-NEXT:    cmovoq %r9, %r102494; SSSE3-NEXT:    movq %rax, %xmm12495; SSSE3-NEXT:    movq %rcx, %rax2496; SSSE3-NEXT:    mulq %rsi2497; SSSE3-NEXT:    movq %rax, %xmm02498; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]2499; SSSE3-NEXT:    movq %r10, %xmm02500; SSSE3-NEXT:    cmovoq %r9, %r82501; SSSE3-NEXT:    movq %r8, %xmm22502; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]2503; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]2504; SSSE3-NEXT:    movdqa %xmm1, (%rdi)2505; SSSE3-NEXT:    retq2506;2507; SSE41-LABEL: umulo_v2i64:2508; SSE41:       # %bb.0:2509; SSE41-NEXT:    movq %xmm0, %rcx2510; SSE41-NEXT:    movq %xmm1, %rsi2511; SSE41-NEXT:    pextrq $1, %xmm0, %rax2512; SSE41-NEXT:    pextrq $1, %xmm1, %rdx2513; SSE41-NEXT:    xorl %r8d, %r8d2514; SSE41-NEXT:    mulq %rdx2515; SSE41-NEXT:    movq $-1, %r92516; SSE41-NEXT:    movl $0, %r10d2517; SSE41-NEXT:    cmovoq %r9, %r102518; SSE41-NEXT:    movq %rax, %xmm02519; SSE41-NEXT:    movq %rcx, %rax2520; SSE41-NEXT:    mulq %rsi2521; SSE41-NEXT:    movq %rax, %xmm12522; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]2523; SSE41-NEXT:    movq %r10, %xmm02524; SSE41-NEXT:    cmovoq %r9, %r82525; SSE41-NEXT:    movq %r8, %xmm22526; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]2527; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]2528; SSE41-NEXT:    movdqa %xmm1, (%rdi)2529; SSE41-NEXT:    retq2530;2531; AVX-LABEL: umulo_v2i64:2532; AVX:       # %bb.0:2533; AVX-NEXT:    vmovq %xmm0, %rcx2534; AVX-NEXT:    vmovq %xmm1, %rsi2535; AVX-NEXT:    vpextrq $1, %xmm0, %rax2536; AVX-NEXT:    vpextrq $1, %xmm1, %rdx2537; AVX-NEXT:    xorl %r8d, %r8d2538; AVX-NEXT:    mulq %rdx2539; AVX-NEXT:    movq $-1, %r92540; AVX-NEXT:    movl $0, %r10d2541; AVX-NEXT:    cmovoq %r9, %r102542; AVX-NEXT:    vmovq %rax, %xmm02543; AVX-NEXT:    movq %rcx, %rax2544; AVX-NEXT:    mulq %rsi2545; AVX-NEXT:    vmovq %rax, %xmm12546; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]2547; AVX-NEXT:    vmovq %r10, %xmm02548; AVX-NEXT:    cmovoq %r9, %r82549; AVX-NEXT:    vmovq %r8, %xmm22550; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]2551; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]2552; AVX-NEXT:    vmovdqa %xmm1, (%rdi)2553; AVX-NEXT:    retq2554;2555; AVX512F-LABEL: umulo_v2i64:2556; AVX512F:       # %bb.0:2557; AVX512F-NEXT:    vmovq %xmm0, %rcx2558; AVX512F-NEXT:    vmovq %xmm1, %rsi2559; AVX512F-NEXT:    vpextrq $1, %xmm0, %rax2560; AVX512F-NEXT:    vpextrq $1, %xmm1, %rdx2561; AVX512F-NEXT:    mulq %rdx2562; AVX512F-NEXT:    seto %r8b2563; AVX512F-NEXT:    vmovq %rax, %xmm02564; AVX512F-NEXT:    movq %rcx, %rax2565; AVX512F-NEXT:    mulq %rsi2566; AVX512F-NEXT:    vmovq %rax, %xmm12567; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]2568; AVX512F-NEXT:    seto %al2569; AVX512F-NEXT:    andl $1, %eax2570; AVX512F-NEXT:    kmovw %eax, %k02571; AVX512F-NEXT:    kmovw %r8d, %k12572; AVX512F-NEXT:    kshiftlw $15, %k1, %k12573; AVX512F-NEXT:    kshiftrw $14, %k1, %k12574; AVX512F-NEXT:    korw %k1, %k0, %k12575; AVX512F-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm02576; AVX512F-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}2577; AVX512F-NEXT:    vmovdqa %xmm1, (%rdi)2578; AVX512F-NEXT:    retq2579;2580; AVX512BW-LABEL: umulo_v2i64:2581; AVX512BW:       # %bb.0:2582; AVX512BW-NEXT:    vmovq %xmm0, %rcx2583; AVX512BW-NEXT:    vmovq %xmm1, %rsi2584; AVX512BW-NEXT:    vpextrq $1, %xmm0, %rax2585; AVX512BW-NEXT:    vpextrq $1, %xmm1, %rdx2586; AVX512BW-NEXT:    mulq %rdx2587; AVX512BW-NEXT:    seto %r8b2588; AVX512BW-NEXT:    vmovq %rax, %xmm02589; AVX512BW-NEXT:    movq %rcx, %rax2590; AVX512BW-NEXT:    mulq %rsi2591; AVX512BW-NEXT:    vmovq %rax, %xmm12592; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]2593; AVX512BW-NEXT:    seto %al2594; AVX512BW-NEXT:    andl $1, %eax2595; AVX512BW-NEXT:    kmovw %eax, %k02596; AVX512BW-NEXT:    kmovd %r8d, %k12597; AVX512BW-NEXT:    kshiftlw $15, %k1, %k12598; AVX512BW-NEXT:    kshiftrw $14, %k1, %k12599; AVX512BW-NEXT:    korw %k1, %k0, %k12600; AVX512BW-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm02601; AVX512BW-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}2602; AVX512BW-NEXT:    vmovdqa %xmm1, (%rdi)2603; AVX512BW-NEXT:    retq2604  %t = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)2605  %val = extractvalue {<2 x i64>, <2 x i1>} %t, 02606  %obit = extractvalue {<2 x i64>, <2 x i1>} %t, 12607  %res = sext <2 x i1> %obit to <2 x i32>2608  store <2 x i64> %val, ptr %p22609  ret <2 x i32> %res2610}2611 2612define <4 x i32> @umulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, ptr %p2) nounwind {2613; SSE2-LABEL: umulo_v4i24:2614; SSE2:       # %bb.0:2615; SSE2-NEXT:    movdqa %xmm0, %xmm22616; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]2617; SSE2-NEXT:    pand %xmm0, %xmm12618; SSE2-NEXT:    pand %xmm0, %xmm22619; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]2620; SSE2-NEXT:    pmuludq %xmm1, %xmm22621; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3]2622; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]2623; SSE2-NEXT:    pmuludq %xmm0, %xmm12624; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]2625; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]2626; SSE2-NEXT:    pxor %xmm4, %xmm42627; SSE2-NEXT:    pcmpeqd %xmm4, %xmm32628; SSE2-NEXT:    pcmpeqd %xmm5, %xmm52629; SSE2-NEXT:    pxor %xmm3, %xmm52630; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]2631; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3]2632; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]2633; SSE2-NEXT:    psrld $24, %xmm02634; SSE2-NEXT:    pcmpgtd %xmm4, %xmm02635; SSE2-NEXT:    por %xmm5, %xmm02636; SSE2-NEXT:    movd %xmm2, %eax2637; SSE2-NEXT:    movw %ax, (%rdi)2638; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]2639; SSE2-NEXT:    movd %xmm2, %ecx2640; SSE2-NEXT:    movw %cx, 6(%rdi)2641; SSE2-NEXT:    movd %xmm1, %edx2642; SSE2-NEXT:    movw %dx, 3(%rdi)2643; SSE2-NEXT:    shrl $16, %eax2644; SSE2-NEXT:    movb %al, 2(%rdi)2645; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]2646; SSE2-NEXT:    movd %xmm1, %eax2647; SSE2-NEXT:    movw %ax, 9(%rdi)2648; SSE2-NEXT:    shrl $16, %ecx2649; SSE2-NEXT:    movb %cl, 8(%rdi)2650; SSE2-NEXT:    shrl $16, %edx2651; SSE2-NEXT:    movb %dl, 5(%rdi)2652; SSE2-NEXT:    shrl $16, %eax2653; SSE2-NEXT:    movb %al, 11(%rdi)2654; SSE2-NEXT:    retq2655;2656; SSSE3-LABEL: umulo_v4i24:2657; SSSE3:       # %bb.0:2658; SSSE3-NEXT:    movdqa %xmm0, %xmm22659; SSSE3-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]2660; SSSE3-NEXT:    pand %xmm0, %xmm12661; SSSE3-NEXT:    pand %xmm0, %xmm22662; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]2663; SSSE3-NEXT:    pmuludq %xmm1, %xmm22664; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,3,2,3]2665; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]2666; SSSE3-NEXT:    pmuludq %xmm0, %xmm12667; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]2668; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]2669; SSSE3-NEXT:    pxor %xmm4, %xmm42670; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm32671; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm52672; SSSE3-NEXT:    pxor %xmm3, %xmm52673; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]2674; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3]2675; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]2676; SSSE3-NEXT:    psrld $24, %xmm02677; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm02678; SSSE3-NEXT:    por %xmm5, %xmm02679; SSSE3-NEXT:    movd %xmm2, %eax2680; SSSE3-NEXT:    movw %ax, (%rdi)2681; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]2682; SSSE3-NEXT:    movd %xmm2, %ecx2683; SSSE3-NEXT:    movw %cx, 6(%rdi)2684; SSSE3-NEXT:    movd %xmm1, %edx2685; SSSE3-NEXT:    movw %dx, 3(%rdi)2686; SSSE3-NEXT:    shrl $16, %eax2687; SSSE3-NEXT:    movb %al, 2(%rdi)2688; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2]2689; SSSE3-NEXT:    movd %xmm1, %eax2690; SSSE3-NEXT:    movw %ax, 9(%rdi)2691; SSSE3-NEXT:    shrl $16, %ecx2692; SSSE3-NEXT:    movb %cl, 8(%rdi)2693; SSSE3-NEXT:    shrl $16, %edx2694; SSSE3-NEXT:    movb %dl, 5(%rdi)2695; SSSE3-NEXT:    shrl $16, %eax2696; SSSE3-NEXT:    movb %al, 11(%rdi)2697; SSSE3-NEXT:    retq2698;2699; SSE41-LABEL: umulo_v4i24:2700; SSE41:       # %bb.0:2701; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]2702; SSE41-NEXT:    pand %xmm2, %xmm02703; SSE41-NEXT:    pand %xmm2, %xmm12704; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]2705; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]2706; SSE41-NEXT:    pmuludq %xmm2, %xmm32707; SSE41-NEXT:    movdqa %xmm0, %xmm22708; SSE41-NEXT:    pmuludq %xmm1, %xmm22709; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]2710; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]2711; SSE41-NEXT:    pxor %xmm3, %xmm32712; SSE41-NEXT:    pcmpeqd %xmm3, %xmm22713; SSE41-NEXT:    pcmpeqd %xmm4, %xmm42714; SSE41-NEXT:    pxor %xmm2, %xmm42715; SSE41-NEXT:    pmulld %xmm0, %xmm12716; SSE41-NEXT:    pextrd $3, %xmm1, %eax2717; SSE41-NEXT:    pextrd $2, %xmm1, %ecx2718; SSE41-NEXT:    pextrd $1, %xmm1, %edx2719; SSE41-NEXT:    movd %xmm1, %esi2720; SSE41-NEXT:    movdqa %xmm1, %xmm02721; SSE41-NEXT:    psrld $24, %xmm02722; SSE41-NEXT:    pcmpgtd %xmm3, %xmm02723; SSE41-NEXT:    por %xmm4, %xmm02724; SSE41-NEXT:    movw %ax, 9(%rdi)2725; SSE41-NEXT:    movw %cx, 6(%rdi)2726; SSE41-NEXT:    movw %dx, 3(%rdi)2727; SSE41-NEXT:    movw %si, (%rdi)2728; SSE41-NEXT:    shrl $16, %eax2729; SSE41-NEXT:    movb %al, 11(%rdi)2730; SSE41-NEXT:    shrl $16, %ecx2731; SSE41-NEXT:    movb %cl, 8(%rdi)2732; SSE41-NEXT:    shrl $16, %edx2733; SSE41-NEXT:    movb %dl, 5(%rdi)2734; SSE41-NEXT:    shrl $16, %esi2735; SSE41-NEXT:    movb %sil, 2(%rdi)2736; SSE41-NEXT:    retq2737;2738; AVX1-LABEL: umulo_v4i24:2739; AVX1:       # %bb.0:2740; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]2741; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm02742; AVX1-NEXT:    vandps %xmm2, %xmm1, %xmm12743; AVX1-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[1,1,3,3]2744; AVX1-NEXT:    vshufps {{.*#+}} xmm3 = xmm0[1,1,3,3]2745; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm22746; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm32747; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]2748; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]2749; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm32750; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm22751; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm42752; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm22753; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm12754; AVX1-NEXT:    vpsrld $24, %xmm1, %xmm02755; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm0, %xmm02756; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm02757; AVX1-NEXT:    vpextrd $3, %xmm1, %eax2758; AVX1-NEXT:    movw %ax, 9(%rdi)2759; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx2760; AVX1-NEXT:    movw %cx, 6(%rdi)2761; AVX1-NEXT:    vpextrd $1, %xmm1, %edx2762; AVX1-NEXT:    movw %dx, 3(%rdi)2763; AVX1-NEXT:    vmovd %xmm1, %esi2764; AVX1-NEXT:    movw %si, (%rdi)2765; AVX1-NEXT:    shrl $16, %eax2766; AVX1-NEXT:    movb %al, 11(%rdi)2767; AVX1-NEXT:    shrl $16, %ecx2768; AVX1-NEXT:    movb %cl, 8(%rdi)2769; AVX1-NEXT:    shrl $16, %edx2770; AVX1-NEXT:    movb %dl, 5(%rdi)2771; AVX1-NEXT:    shrl $16, %esi2772; AVX1-NEXT:    movb %sil, 2(%rdi)2773; AVX1-NEXT:    retq2774;2775; AVX2-LABEL: umulo_v4i24:2776; AVX2:       # %bb.0:2777; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]2778; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm02779; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm12780; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]2781; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]2782; AVX2-NEXT:    vpmuludq %xmm2, %xmm3, %xmm22783; AVX2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm32784; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]2785; AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]2786; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm32787; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm22788; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm42789; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm22790; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm12791; AVX2-NEXT:    vpsrld $24, %xmm1, %xmm02792; AVX2-NEXT:    vpcmpgtd %xmm3, %xmm0, %xmm02793; AVX2-NEXT:    vpor %xmm2, %xmm0, %xmm02794; AVX2-NEXT:    vpextrd $3, %xmm1, %eax2795; AVX2-NEXT:    movw %ax, 9(%rdi)2796; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx2797; AVX2-NEXT:    movw %cx, 6(%rdi)2798; AVX2-NEXT:    vpextrd $1, %xmm1, %edx2799; AVX2-NEXT:    movw %dx, 3(%rdi)2800; AVX2-NEXT:    vmovd %xmm1, %esi2801; AVX2-NEXT:    movw %si, (%rdi)2802; AVX2-NEXT:    shrl $16, %eax2803; AVX2-NEXT:    movb %al, 11(%rdi)2804; AVX2-NEXT:    shrl $16, %ecx2805; AVX2-NEXT:    movb %cl, 8(%rdi)2806; AVX2-NEXT:    shrl $16, %edx2807; AVX2-NEXT:    movb %dl, 5(%rdi)2808; AVX2-NEXT:    shrl $16, %esi2809; AVX2-NEXT:    movb %sil, 2(%rdi)2810; AVX2-NEXT:    retq2811;2812; AVX512-LABEL: umulo_v4i24:2813; AVX512:       # %bb.0:2814; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]2815; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm12816; AVX512-NEXT:    vpand %xmm2, %xmm0, %xmm02817; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm22818; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]2819; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]2820; AVX512-NEXT:    vpmuludq %xmm3, %xmm4, %xmm32821; AVX512-NEXT:    vpmovsxbd {{.*#+}} xmm4 = [1,5,3,7]2822; AVX512-NEXT:    vpermi2d %xmm3, %xmm2, %xmm42823; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm12824; AVX512-NEXT:    vpsrld $24, %xmm1, %xmm02825; AVX512-NEXT:    vpor %xmm4, %xmm0, %xmm02826; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k12827; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm02828; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}2829; AVX512-NEXT:    vpextrd $3, %xmm1, %eax2830; AVX512-NEXT:    movw %ax, 9(%rdi)2831; AVX512-NEXT:    vpextrd $2, %xmm1, %ecx2832; AVX512-NEXT:    movw %cx, 6(%rdi)2833; AVX512-NEXT:    vpextrd $1, %xmm1, %edx2834; AVX512-NEXT:    movw %dx, 3(%rdi)2835; AVX512-NEXT:    vmovd %xmm1, %esi2836; AVX512-NEXT:    movw %si, (%rdi)2837; AVX512-NEXT:    shrl $16, %eax2838; AVX512-NEXT:    movb %al, 11(%rdi)2839; AVX512-NEXT:    shrl $16, %ecx2840; AVX512-NEXT:    movb %cl, 8(%rdi)2841; AVX512-NEXT:    shrl $16, %edx2842; AVX512-NEXT:    movb %dl, 5(%rdi)2843; AVX512-NEXT:    shrl $16, %esi2844; AVX512-NEXT:    movb %sil, 2(%rdi)2845; AVX512-NEXT:    retq2846  %t = call {<4 x i24>, <4 x i1>} @llvm.umul.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)2847  %val = extractvalue {<4 x i24>, <4 x i1>} %t, 02848  %obit = extractvalue {<4 x i24>, <4 x i1>} %t, 12849  %res = sext <4 x i1> %obit to <4 x i32>2850  store <4 x i24> %val, ptr %p22851  ret <4 x i32> %res2852}2853 2854define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, ptr %p2) nounwind {2855; SSE-LABEL: umulo_v4i1:2856; SSE:       # %bb.0:2857; SSE-NEXT:    pand %xmm1, %xmm02858; SSE-NEXT:    pslld $31, %xmm02859; SSE-NEXT:    movmskps %xmm0, %eax2860; SSE-NEXT:    movb %al, (%rdi)2861; SSE-NEXT:    xorps %xmm0, %xmm02862; SSE-NEXT:    retq2863;2864; AVX-LABEL: umulo_v4i1:2865; AVX:       # %bb.0:2866; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm02867; AVX-NEXT:    vpslld $31, %xmm0, %xmm02868; AVX-NEXT:    vmovmskps %xmm0, %eax2869; AVX-NEXT:    movb %al, (%rdi)2870; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm02871; AVX-NEXT:    retq2872;2873; AVX512F-LABEL: umulo_v4i1:2874; AVX512F:       # %bb.0:2875; AVX512F-NEXT:    vpand %xmm1, %xmm0, %xmm02876; AVX512F-NEXT:    vpslld $31, %xmm0, %xmm02877; AVX512F-NEXT:    vptestmd %xmm0, %xmm0, %k02878; AVX512F-NEXT:    kmovw %k0, %eax2879; AVX512F-NEXT:    movb %al, (%rdi)2880; AVX512F-NEXT:    vpxor %xmm0, %xmm0, %xmm02881; AVX512F-NEXT:    retq2882;2883; AVX512BW-LABEL: umulo_v4i1:2884; AVX512BW:       # %bb.0:2885; AVX512BW-NEXT:    vpand %xmm1, %xmm0, %xmm02886; AVX512BW-NEXT:    vpslld $31, %xmm0, %xmm02887; AVX512BW-NEXT:    vptestmd %xmm0, %xmm0, %k02888; AVX512BW-NEXT:    kmovd %k0, %eax2889; AVX512BW-NEXT:    movb %al, (%rdi)2890; AVX512BW-NEXT:    vpxor %xmm0, %xmm0, %xmm02891; AVX512BW-NEXT:    retq2892  %t = call {<4 x i1>, <4 x i1>} @llvm.umul.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)2893  %val = extractvalue {<4 x i1>, <4 x i1>} %t, 02894  %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 12895  %res = sext <4 x i1> %obit to <4 x i32>2896  store <4 x i1> %val, ptr %p22897  ret <4 x i32> %res2898}2899 2900define <2 x i32> @umulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, ptr %p2) nounwind {2901; SSE2-LABEL: umulo_v2i128:2902; SSE2:       # %bb.0:2903; SSE2-NEXT:    pushq %rbp2904; SSE2-NEXT:    pushq %r152905; SSE2-NEXT:    pushq %r142906; SSE2-NEXT:    pushq %r122907; SSE2-NEXT:    pushq %rbx2908; SSE2-NEXT:    movq %r9, %r112909; SSE2-NEXT:    movq %rcx, %r102910; SSE2-NEXT:    movq %rdx, %rcx2911; SSE2-NEXT:    movq %rsi, %rax2912; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rbx2913; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r142914; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r92915; SSE2-NEXT:    testq %r11, %r112916; SSE2-NEXT:    setne %dl2917; SSE2-NEXT:    testq %rsi, %rsi2918; SSE2-NEXT:    setne %bpl2919; SSE2-NEXT:    andb %dl, %bpl2920; SSE2-NEXT:    mulq %r82921; SSE2-NEXT:    movq %rax, %rsi2922; SSE2-NEXT:    seto %r15b2923; SSE2-NEXT:    movq %r11, %rax2924; SSE2-NEXT:    mulq %rdi2925; SSE2-NEXT:    seto %r12b2926; SSE2-NEXT:    orb %r15b, %r12b2927; SSE2-NEXT:    orb %bpl, %r12b2928; SSE2-NEXT:    leaq (%rsi,%rax), %r112929; SSE2-NEXT:    movq %rdi, %rax2930; SSE2-NEXT:    mulq %r82931; SSE2-NEXT:    movq %rax, %rdi2932; SSE2-NEXT:    movq %rdx, %rsi2933; SSE2-NEXT:    addq %r11, %rsi2934; SSE2-NEXT:    setb %r11b2935; SSE2-NEXT:    orb %r12b, %r11b2936; SSE2-NEXT:    testq %r9, %r92937; SSE2-NEXT:    setne %al2938; SSE2-NEXT:    testq %r10, %r102939; SSE2-NEXT:    setne %bpl2940; SSE2-NEXT:    andb %al, %bpl2941; SSE2-NEXT:    movq %r10, %rax2942; SSE2-NEXT:    mulq %r142943; SSE2-NEXT:    movq %rax, %r82944; SSE2-NEXT:    seto %r10b2945; SSE2-NEXT:    movq %r9, %rax2946; SSE2-NEXT:    mulq %rcx2947; SSE2-NEXT:    seto %r9b2948; SSE2-NEXT:    orb %r10b, %r9b2949; SSE2-NEXT:    orb %bpl, %r9b2950; SSE2-NEXT:    addq %rax, %r82951; SSE2-NEXT:    movq %rcx, %rax2952; SSE2-NEXT:    mulq %r142953; SSE2-NEXT:    addq %r8, %rdx2954; SSE2-NEXT:    setb %cl2955; SSE2-NEXT:    orb %r9b, %cl2956; SSE2-NEXT:    movzbl %cl, %ecx2957; SSE2-NEXT:    negl %ecx2958; SSE2-NEXT:    movd %ecx, %xmm12959; SSE2-NEXT:    movzbl %r11b, %ecx2960; SSE2-NEXT:    negl %ecx2961; SSE2-NEXT:    movd %ecx, %xmm02962; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]2963; SSE2-NEXT:    movq %rax, 16(%rbx)2964; SSE2-NEXT:    movq %rdi, (%rbx)2965; SSE2-NEXT:    movq %rdx, 24(%rbx)2966; SSE2-NEXT:    movq %rsi, 8(%rbx)2967; SSE2-NEXT:    popq %rbx2968; SSE2-NEXT:    popq %r122969; SSE2-NEXT:    popq %r142970; SSE2-NEXT:    popq %r152971; SSE2-NEXT:    popq %rbp2972; SSE2-NEXT:    retq2973;2974; SSSE3-LABEL: umulo_v2i128:2975; SSSE3:       # %bb.0:2976; SSSE3-NEXT:    pushq %rbp2977; SSSE3-NEXT:    pushq %r152978; SSSE3-NEXT:    pushq %r142979; SSSE3-NEXT:    pushq %r122980; SSSE3-NEXT:    pushq %rbx2981; SSSE3-NEXT:    movq %r9, %r112982; SSSE3-NEXT:    movq %rcx, %r102983; SSSE3-NEXT:    movq %rdx, %rcx2984; SSSE3-NEXT:    movq %rsi, %rax2985; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rbx2986; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r142987; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r92988; SSSE3-NEXT:    testq %r11, %r112989; SSSE3-NEXT:    setne %dl2990; SSSE3-NEXT:    testq %rsi, %rsi2991; SSSE3-NEXT:    setne %bpl2992; SSSE3-NEXT:    andb %dl, %bpl2993; SSSE3-NEXT:    mulq %r82994; SSSE3-NEXT:    movq %rax, %rsi2995; SSSE3-NEXT:    seto %r15b2996; SSSE3-NEXT:    movq %r11, %rax2997; SSSE3-NEXT:    mulq %rdi2998; SSSE3-NEXT:    seto %r12b2999; SSSE3-NEXT:    orb %r15b, %r12b3000; SSSE3-NEXT:    orb %bpl, %r12b3001; SSSE3-NEXT:    leaq (%rsi,%rax), %r113002; SSSE3-NEXT:    movq %rdi, %rax3003; SSSE3-NEXT:    mulq %r83004; SSSE3-NEXT:    movq %rax, %rdi3005; SSSE3-NEXT:    movq %rdx, %rsi3006; SSSE3-NEXT:    addq %r11, %rsi3007; SSSE3-NEXT:    setb %r11b3008; SSSE3-NEXT:    orb %r12b, %r11b3009; SSSE3-NEXT:    testq %r9, %r93010; SSSE3-NEXT:    setne %al3011; SSSE3-NEXT:    testq %r10, %r103012; SSSE3-NEXT:    setne %bpl3013; SSSE3-NEXT:    andb %al, %bpl3014; SSSE3-NEXT:    movq %r10, %rax3015; SSSE3-NEXT:    mulq %r143016; SSSE3-NEXT:    movq %rax, %r83017; SSSE3-NEXT:    seto %r10b3018; SSSE3-NEXT:    movq %r9, %rax3019; SSSE3-NEXT:    mulq %rcx3020; SSSE3-NEXT:    seto %r9b3021; SSSE3-NEXT:    orb %r10b, %r9b3022; SSSE3-NEXT:    orb %bpl, %r9b3023; SSSE3-NEXT:    addq %rax, %r83024; SSSE3-NEXT:    movq %rcx, %rax3025; SSSE3-NEXT:    mulq %r143026; SSSE3-NEXT:    addq %r8, %rdx3027; SSSE3-NEXT:    setb %cl3028; SSSE3-NEXT:    orb %r9b, %cl3029; SSSE3-NEXT:    movzbl %cl, %ecx3030; SSSE3-NEXT:    negl %ecx3031; SSSE3-NEXT:    movd %ecx, %xmm13032; SSSE3-NEXT:    movzbl %r11b, %ecx3033; SSSE3-NEXT:    negl %ecx3034; SSSE3-NEXT:    movd %ecx, %xmm03035; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]3036; SSSE3-NEXT:    movq %rax, 16(%rbx)3037; SSSE3-NEXT:    movq %rdi, (%rbx)3038; SSSE3-NEXT:    movq %rdx, 24(%rbx)3039; SSSE3-NEXT:    movq %rsi, 8(%rbx)3040; SSSE3-NEXT:    popq %rbx3041; SSSE3-NEXT:    popq %r123042; SSSE3-NEXT:    popq %r143043; SSSE3-NEXT:    popq %r153044; SSSE3-NEXT:    popq %rbp3045; SSSE3-NEXT:    retq3046;3047; SSE41-LABEL: umulo_v2i128:3048; SSE41:       # %bb.0:3049; SSE41-NEXT:    pushq %rbp3050; SSE41-NEXT:    pushq %r153051; SSE41-NEXT:    pushq %r143052; SSE41-NEXT:    pushq %r123053; SSE41-NEXT:    pushq %rbx3054; SSE41-NEXT:    movq %r9, %r113055; SSE41-NEXT:    movq %rcx, %r103056; SSE41-NEXT:    movq %rdx, %rcx3057; SSE41-NEXT:    movq %rsi, %rax3058; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rbx3059; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r143060; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r93061; SSE41-NEXT:    testq %r11, %r113062; SSE41-NEXT:    setne %dl3063; SSE41-NEXT:    testq %rsi, %rsi3064; SSE41-NEXT:    setne %bpl3065; SSE41-NEXT:    andb %dl, %bpl3066; SSE41-NEXT:    mulq %r83067; SSE41-NEXT:    movq %rax, %rsi3068; SSE41-NEXT:    seto %r15b3069; SSE41-NEXT:    movq %r11, %rax3070; SSE41-NEXT:    mulq %rdi3071; SSE41-NEXT:    seto %r12b3072; SSE41-NEXT:    orb %r15b, %r12b3073; SSE41-NEXT:    orb %bpl, %r12b3074; SSE41-NEXT:    leaq (%rsi,%rax), %r113075; SSE41-NEXT:    movq %rdi, %rax3076; SSE41-NEXT:    mulq %r83077; SSE41-NEXT:    movq %rax, %rdi3078; SSE41-NEXT:    movq %rdx, %rsi3079; SSE41-NEXT:    addq %r11, %rsi3080; SSE41-NEXT:    setb %r11b3081; SSE41-NEXT:    orb %r12b, %r11b3082; SSE41-NEXT:    testq %r9, %r93083; SSE41-NEXT:    setne %al3084; SSE41-NEXT:    testq %r10, %r103085; SSE41-NEXT:    setne %bpl3086; SSE41-NEXT:    andb %al, %bpl3087; SSE41-NEXT:    movq %r10, %rax3088; SSE41-NEXT:    mulq %r143089; SSE41-NEXT:    movq %rax, %r83090; SSE41-NEXT:    seto %r10b3091; SSE41-NEXT:    movq %r9, %rax3092; SSE41-NEXT:    mulq %rcx3093; SSE41-NEXT:    seto %r9b3094; SSE41-NEXT:    orb %r10b, %r9b3095; SSE41-NEXT:    orb %bpl, %r9b3096; SSE41-NEXT:    addq %rax, %r83097; SSE41-NEXT:    movq %rcx, %rax3098; SSE41-NEXT:    mulq %r143099; SSE41-NEXT:    addq %r8, %rdx3100; SSE41-NEXT:    setb %cl3101; SSE41-NEXT:    orb %r9b, %cl3102; SSE41-NEXT:    movzbl %cl, %ecx3103; SSE41-NEXT:    negl %ecx3104; SSE41-NEXT:    movzbl %r11b, %r8d3105; SSE41-NEXT:    negl %r8d3106; SSE41-NEXT:    movd %r8d, %xmm03107; SSE41-NEXT:    pinsrd $1, %ecx, %xmm03108; SSE41-NEXT:    movq %rax, 16(%rbx)3109; SSE41-NEXT:    movq %rdi, (%rbx)3110; SSE41-NEXT:    movq %rdx, 24(%rbx)3111; SSE41-NEXT:    movq %rsi, 8(%rbx)3112; SSE41-NEXT:    popq %rbx3113; SSE41-NEXT:    popq %r123114; SSE41-NEXT:    popq %r143115; SSE41-NEXT:    popq %r153116; SSE41-NEXT:    popq %rbp3117; SSE41-NEXT:    retq3118;3119; AVX-LABEL: umulo_v2i128:3120; AVX:       # %bb.0:3121; AVX-NEXT:    pushq %rbp3122; AVX-NEXT:    pushq %r153123; AVX-NEXT:    pushq %r143124; AVX-NEXT:    pushq %r123125; AVX-NEXT:    pushq %rbx3126; AVX-NEXT:    movq %r9, %r113127; AVX-NEXT:    movq %rcx, %r103128; AVX-NEXT:    movq %rdx, %rcx3129; AVX-NEXT:    movq %rsi, %rax3130; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rbx3131; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r143132; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r93133; AVX-NEXT:    testq %r11, %r113134; AVX-NEXT:    setne %dl3135; AVX-NEXT:    testq %rsi, %rsi3136; AVX-NEXT:    setne %bpl3137; AVX-NEXT:    andb %dl, %bpl3138; AVX-NEXT:    mulq %r83139; AVX-NEXT:    movq %rax, %rsi3140; AVX-NEXT:    seto %r15b3141; AVX-NEXT:    movq %r11, %rax3142; AVX-NEXT:    mulq %rdi3143; AVX-NEXT:    seto %r12b3144; AVX-NEXT:    orb %r15b, %r12b3145; AVX-NEXT:    orb %bpl, %r12b3146; AVX-NEXT:    leaq (%rsi,%rax), %r113147; AVX-NEXT:    movq %rdi, %rax3148; AVX-NEXT:    mulq %r83149; AVX-NEXT:    movq %rax, %rdi3150; AVX-NEXT:    movq %rdx, %rsi3151; AVX-NEXT:    addq %r11, %rsi3152; AVX-NEXT:    setb %r11b3153; AVX-NEXT:    orb %r12b, %r11b3154; AVX-NEXT:    testq %r9, %r93155; AVX-NEXT:    setne %al3156; AVX-NEXT:    testq %r10, %r103157; AVX-NEXT:    setne %bpl3158; AVX-NEXT:    andb %al, %bpl3159; AVX-NEXT:    movq %r10, %rax3160; AVX-NEXT:    mulq %r143161; AVX-NEXT:    movq %rax, %r83162; AVX-NEXT:    seto %r10b3163; AVX-NEXT:    movq %r9, %rax3164; AVX-NEXT:    mulq %rcx3165; AVX-NEXT:    seto %r9b3166; AVX-NEXT:    orb %r10b, %r9b3167; AVX-NEXT:    orb %bpl, %r9b3168; AVX-NEXT:    addq %rax, %r83169; AVX-NEXT:    movq %rcx, %rax3170; AVX-NEXT:    mulq %r143171; AVX-NEXT:    addq %r8, %rdx3172; AVX-NEXT:    setb %cl3173; AVX-NEXT:    orb %r9b, %cl3174; AVX-NEXT:    movzbl %cl, %ecx3175; AVX-NEXT:    negl %ecx3176; AVX-NEXT:    movzbl %r11b, %r8d3177; AVX-NEXT:    negl %r8d3178; AVX-NEXT:    vmovd %r8d, %xmm03179; AVX-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm03180; AVX-NEXT:    movq %rax, 16(%rbx)3181; AVX-NEXT:    movq %rdi, (%rbx)3182; AVX-NEXT:    movq %rdx, 24(%rbx)3183; AVX-NEXT:    movq %rsi, 8(%rbx)3184; AVX-NEXT:    popq %rbx3185; AVX-NEXT:    popq %r123186; AVX-NEXT:    popq %r143187; AVX-NEXT:    popq %r153188; AVX-NEXT:    popq %rbp3189; AVX-NEXT:    retq3190;3191; AVX512F-LABEL: umulo_v2i128:3192; AVX512F:       # %bb.0:3193; AVX512F-NEXT:    pushq %rbp3194; AVX512F-NEXT:    pushq %r153195; AVX512F-NEXT:    pushq %r143196; AVX512F-NEXT:    pushq %r123197; AVX512F-NEXT:    pushq %rbx3198; AVX512F-NEXT:    movq %rcx, %rax3199; AVX512F-NEXT:    movq %rdx, %rcx3200; AVX512F-NEXT:    movq %rsi, %r103201; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rbx3202; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r143203; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rsi3204; AVX512F-NEXT:    testq %rsi, %rsi3205; AVX512F-NEXT:    setne %dl3206; AVX512F-NEXT:    testq %rax, %rax3207; AVX512F-NEXT:    setne %bpl3208; AVX512F-NEXT:    andb %dl, %bpl3209; AVX512F-NEXT:    mulq %r143210; AVX512F-NEXT:    movq %rax, %r113211; AVX512F-NEXT:    seto %r15b3212; AVX512F-NEXT:    movq %rsi, %rax3213; AVX512F-NEXT:    mulq %rcx3214; AVX512F-NEXT:    seto %r12b3215; AVX512F-NEXT:    orb %r15b, %r12b3216; AVX512F-NEXT:    orb %bpl, %r12b3217; AVX512F-NEXT:    addq %rax, %r113218; AVX512F-NEXT:    movq %rcx, %rax3219; AVX512F-NEXT:    mulq %r143220; AVX512F-NEXT:    movq %rax, %rsi3221; AVX512F-NEXT:    movq %rdx, %rcx3222; AVX512F-NEXT:    addq %r11, %rcx3223; AVX512F-NEXT:    setb %al3224; AVX512F-NEXT:    orb %r12b, %al3225; AVX512F-NEXT:    kmovw %eax, %k03226; AVX512F-NEXT:    testq %r9, %r93227; AVX512F-NEXT:    setne %al3228; AVX512F-NEXT:    testq %r10, %r103229; AVX512F-NEXT:    setne %r11b3230; AVX512F-NEXT:    andb %al, %r11b3231; AVX512F-NEXT:    movq %r10, %rax3232; AVX512F-NEXT:    mulq %r83233; AVX512F-NEXT:    movq %rax, %r103234; AVX512F-NEXT:    seto %bpl3235; AVX512F-NEXT:    movq %r9, %rax3236; AVX512F-NEXT:    mulq %rdi3237; AVX512F-NEXT:    seto %r9b3238; AVX512F-NEXT:    orb %bpl, %r9b3239; AVX512F-NEXT:    orb %r11b, %r9b3240; AVX512F-NEXT:    addq %rax, %r103241; AVX512F-NEXT:    movq %rdi, %rax3242; AVX512F-NEXT:    mulq %r83243; AVX512F-NEXT:    addq %r10, %rdx3244; AVX512F-NEXT:    setb %dil3245; AVX512F-NEXT:    orb %r9b, %dil3246; AVX512F-NEXT:    andl $1, %edi3247; AVX512F-NEXT:    kmovw %edi, %k13248; AVX512F-NEXT:    kshiftlw $1, %k0, %k03249; AVX512F-NEXT:    korw %k0, %k1, %k13250; AVX512F-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm03251; AVX512F-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}3252; AVX512F-NEXT:    movq %rsi, 16(%rbx)3253; AVX512F-NEXT:    movq %rax, (%rbx)3254; AVX512F-NEXT:    movq %rcx, 24(%rbx)3255; AVX512F-NEXT:    movq %rdx, 8(%rbx)3256; AVX512F-NEXT:    popq %rbx3257; AVX512F-NEXT:    popq %r123258; AVX512F-NEXT:    popq %r143259; AVX512F-NEXT:    popq %r153260; AVX512F-NEXT:    popq %rbp3261; AVX512F-NEXT:    retq3262;3263; AVX512BW-LABEL: umulo_v2i128:3264; AVX512BW:       # %bb.0:3265; AVX512BW-NEXT:    pushq %rbp3266; AVX512BW-NEXT:    pushq %r153267; AVX512BW-NEXT:    pushq %r143268; AVX512BW-NEXT:    pushq %r123269; AVX512BW-NEXT:    pushq %rbx3270; AVX512BW-NEXT:    movq %rcx, %rax3271; AVX512BW-NEXT:    movq %rdx, %rcx3272; AVX512BW-NEXT:    movq %rsi, %r103273; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rbx3274; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r143275; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rsi3276; AVX512BW-NEXT:    testq %rsi, %rsi3277; AVX512BW-NEXT:    setne %dl3278; AVX512BW-NEXT:    testq %rax, %rax3279; AVX512BW-NEXT:    setne %bpl3280; AVX512BW-NEXT:    andb %dl, %bpl3281; AVX512BW-NEXT:    mulq %r143282; AVX512BW-NEXT:    movq %rax, %r113283; AVX512BW-NEXT:    seto %r15b3284; AVX512BW-NEXT:    movq %rsi, %rax3285; AVX512BW-NEXT:    mulq %rcx3286; AVX512BW-NEXT:    seto %r12b3287; AVX512BW-NEXT:    orb %r15b, %r12b3288; AVX512BW-NEXT:    orb %bpl, %r12b3289; AVX512BW-NEXT:    addq %rax, %r113290; AVX512BW-NEXT:    movq %rcx, %rax3291; AVX512BW-NEXT:    mulq %r143292; AVX512BW-NEXT:    movq %rax, %rsi3293; AVX512BW-NEXT:    movq %rdx, %rcx3294; AVX512BW-NEXT:    addq %r11, %rcx3295; AVX512BW-NEXT:    setb %al3296; AVX512BW-NEXT:    orb %r12b, %al3297; AVX512BW-NEXT:    kmovd %eax, %k03298; AVX512BW-NEXT:    testq %r9, %r93299; AVX512BW-NEXT:    setne %al3300; AVX512BW-NEXT:    testq %r10, %r103301; AVX512BW-NEXT:    setne %r11b3302; AVX512BW-NEXT:    andb %al, %r11b3303; AVX512BW-NEXT:    movq %r10, %rax3304; AVX512BW-NEXT:    mulq %r83305; AVX512BW-NEXT:    movq %rax, %r103306; AVX512BW-NEXT:    seto %bpl3307; AVX512BW-NEXT:    movq %r9, %rax3308; AVX512BW-NEXT:    mulq %rdi3309; AVX512BW-NEXT:    seto %r9b3310; AVX512BW-NEXT:    orb %bpl, %r9b3311; AVX512BW-NEXT:    orb %r11b, %r9b3312; AVX512BW-NEXT:    addq %rax, %r103313; AVX512BW-NEXT:    movq %rdi, %rax3314; AVX512BW-NEXT:    mulq %r83315; AVX512BW-NEXT:    addq %r10, %rdx3316; AVX512BW-NEXT:    setb %dil3317; AVX512BW-NEXT:    orb %r9b, %dil3318; AVX512BW-NEXT:    andl $1, %edi3319; AVX512BW-NEXT:    kmovw %edi, %k13320; AVX512BW-NEXT:    kshiftlw $1, %k0, %k03321; AVX512BW-NEXT:    korw %k0, %k1, %k13322; AVX512BW-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm03323; AVX512BW-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}3324; AVX512BW-NEXT:    movq %rsi, 16(%rbx)3325; AVX512BW-NEXT:    movq %rax, (%rbx)3326; AVX512BW-NEXT:    movq %rcx, 24(%rbx)3327; AVX512BW-NEXT:    movq %rdx, 8(%rbx)3328; AVX512BW-NEXT:    popq %rbx3329; AVX512BW-NEXT:    popq %r123330; AVX512BW-NEXT:    popq %r143331; AVX512BW-NEXT:    popq %r153332; AVX512BW-NEXT:    popq %rbp3333; AVX512BW-NEXT:    retq3334  %t = call {<2 x i128>, <2 x i1>} @llvm.umul.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)3335  %val = extractvalue {<2 x i128>, <2 x i1>} %t, 03336  %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 13337  %res = sext <2 x i1> %obit to <2 x i32>3338  store <2 x i128> %val, ptr %p23339  ret <2 x i32> %res3340}3341