3936 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE23; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSSE34; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE415; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX16; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX27; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F8; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F9; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW10; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW11 12declare {<1 x i32>, <1 x i1>} @llvm.smul.with.overflow.v1i32(<1 x i32>, <1 x i32>)13declare {<2 x i32>, <2 x i1>} @llvm.smul.with.overflow.v2i32(<2 x i32>, <2 x i32>)14declare {<3 x i32>, <3 x i1>} @llvm.smul.with.overflow.v3i32(<3 x i32>, <3 x i32>)15declare {<4 x i32>, <4 x i1>} @llvm.smul.with.overflow.v4i32(<4 x i32>, <4 x i32>)16declare {<6 x i32>, <6 x i1>} @llvm.smul.with.overflow.v6i32(<6 x i32>, <6 x i32>)17declare {<8 x i32>, <8 x i1>} @llvm.smul.with.overflow.v8i32(<8 x i32>, <8 x i32>)18declare {<16 x i32>, <16 x i1>} @llvm.smul.with.overflow.v16i32(<16 x i32>, <16 x i32>)19 20declare {<16 x i8>, <16 x i1>} @llvm.smul.with.overflow.v16i8(<16 x i8>, <16 x i8>)21declare {<32 x i8>, <32 x i1>} @llvm.smul.with.overflow.v32i8(<32 x i8>, <32 x i8>)22declare {<64 x i8>, <64 x i1>} @llvm.smul.with.overflow.v64i8(<64 x i8>, <64 x i8>)23declare {<8 x i16>, <8 x i1>} @llvm.smul.with.overflow.v8i16(<8 x i16>, <8 x i16>)24declare {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64>, <2 x i64>)25 26declare {<4 x i24>, <4 x i1>} @llvm.smul.with.overflow.v4i24(<4 x i24>, <4 x i24>)27declare {<4 x i1>, <4 x i1>} @llvm.smul.with.overflow.v4i1(<4 x i1>, <4 x i1>)28declare {<2 x i128>, <2 x i1>} @llvm.smul.with.overflow.v2i128(<2 x i128>, <2 x i128>)29 30define <1 x i32> @smulo_v1i32(<1 x i32> %a0, <1 x i32> %a1, ptr %p2) nounwind {31; CHECK-LABEL: smulo_v1i32:32; CHECK: # %bb.0:33; CHECK-NEXT: xorl %eax, %eax34; CHECK-NEXT: imull %esi, %edi35; CHECK-NEXT: seto %al36; CHECK-NEXT: negl %eax37; CHECK-NEXT: movl %edi, (%rdx)38; CHECK-NEXT: retq39 %t = call {<1 x i32>, <1 x i1>} @llvm.smul.with.overflow.v1i32(<1 x i32> %a0, <1 x i32> %a1)40 %val = extractvalue {<1 x i32>, <1 x i1>} %t, 041 %obit = extractvalue {<1 x i32>, <1 x i1>} %t, 142 %res = sext <1 x i1> %obit to <1 x i32>43 store <1 x i32> %val, ptr %p244 ret <1 x i32> %res45}46 47define <2 x i32> @smulo_v2i32(<2 x i32> %a0, <2 x i32> %a1, ptr %p2) nounwind {48; SSE2-LABEL: smulo_v2i32:49; SSE2: # %bb.0:50; SSE2-NEXT: pxor %xmm2, %xmm251; SSE2-NEXT: pxor %xmm3, %xmm352; SSE2-NEXT: pcmpgtd %xmm1, %xmm353; SSE2-NEXT: pand %xmm0, %xmm354; SSE2-NEXT: pcmpgtd %xmm0, %xmm255; SSE2-NEXT: pand %xmm1, %xmm256; SSE2-NEXT: paddd %xmm3, %xmm257; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]58; SSE2-NEXT: pmuludq %xmm1, %xmm059; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]60; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]61; SSE2-NEXT: pmuludq %xmm3, %xmm162; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]63; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]64; SSE2-NEXT: psubd %xmm2, %xmm465; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]66; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]67; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]68; SSE2-NEXT: movq %xmm0, (%rdi)69; SSE2-NEXT: movdqa %xmm0, %xmm170; SSE2-NEXT: psrad $31, %xmm171; SSE2-NEXT: pcmpeqd %xmm4, %xmm172; SSE2-NEXT: pcmpeqd %xmm0, %xmm073; SSE2-NEXT: pxor %xmm1, %xmm074; SSE2-NEXT: retq75;76; SSSE3-LABEL: smulo_v2i32:77; SSSE3: # %bb.0:78; SSSE3-NEXT: pxor %xmm2, %xmm279; SSSE3-NEXT: pxor %xmm3, %xmm380; SSSE3-NEXT: pcmpgtd %xmm1, %xmm381; SSSE3-NEXT: pand %xmm0, %xmm382; SSSE3-NEXT: pcmpgtd %xmm0, %xmm283; SSSE3-NEXT: pand %xmm1, %xmm284; SSSE3-NEXT: paddd %xmm3, %xmm285; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]86; SSSE3-NEXT: pmuludq %xmm1, %xmm087; SSSE3-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]88; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]89; SSSE3-NEXT: pmuludq %xmm3, %xmm190; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]91; SSSE3-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]92; SSSE3-NEXT: psubd %xmm2, %xmm493; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]94; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]95; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]96; SSSE3-NEXT: movq %xmm0, (%rdi)97; SSSE3-NEXT: movdqa %xmm0, %xmm198; SSSE3-NEXT: psrad $31, %xmm199; SSSE3-NEXT: pcmpeqd %xmm4, %xmm1100; SSSE3-NEXT: pcmpeqd %xmm0, %xmm0101; SSSE3-NEXT: pxor %xmm1, %xmm0102; SSSE3-NEXT: retq103;104; SSE41-LABEL: smulo_v2i32:105; SSE41: # %bb.0:106; SSE41-NEXT: pmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero107; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero108; SSE41-NEXT: pmuldq %xmm1, %xmm0109; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]110; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]111; SSE41-NEXT: movq %xmm2, (%rdi)112; SSE41-NEXT: psrad $31, %xmm2113; SSE41-NEXT: pcmpeqd %xmm1, %xmm2114; SSE41-NEXT: pcmpeqd %xmm0, %xmm0115; SSE41-NEXT: pxor %xmm2, %xmm0116; SSE41-NEXT: retq117;118; AVX-LABEL: smulo_v2i32:119; AVX: # %bb.0:120; AVX-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero121; AVX-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero122; AVX-NEXT: vpmuldq %xmm1, %xmm0, %xmm0123; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]124; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]125; AVX-NEXT: vpsrad $31, %xmm2, %xmm0126; AVX-NEXT: vpcmpeqd %xmm0, %xmm1, %xmm0127; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1128; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0129; AVX-NEXT: vmovq %xmm2, (%rdi)130; AVX-NEXT: retq131;132; AVX512-LABEL: smulo_v2i32:133; AVX512: # %bb.0:134; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero135; AVX512-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero136; AVX512-NEXT: vpmuldq %xmm1, %xmm0, %xmm0137; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]138; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]139; AVX512-NEXT: vpsrad $31, %xmm2, %xmm0140; AVX512-NEXT: vpcmpeqd %xmm0, %xmm1, %xmm0141; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~xmm0142; AVX512-NEXT: vmovq %xmm2, (%rdi)143; AVX512-NEXT: retq144 %t = call {<2 x i32>, <2 x i1>} @llvm.smul.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)145 %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0146 %obit = extractvalue {<2 x i32>, <2 x i1>} %t, 1147 %res = sext <2 x i1> %obit to <2 x i32>148 store <2 x i32> %val, ptr %p2149 ret <2 x i32> %res150}151 152define <3 x i32> @smulo_v3i32(<3 x i32> %a0, <3 x i32> %a1, ptr %p2) nounwind {153; SSE2-LABEL: smulo_v3i32:154; SSE2: # %bb.0:155; SSE2-NEXT: pxor %xmm2, %xmm2156; SSE2-NEXT: pxor %xmm3, %xmm3157; SSE2-NEXT: pcmpgtd %xmm1, %xmm3158; SSE2-NEXT: pand %xmm0, %xmm3159; SSE2-NEXT: pcmpgtd %xmm0, %xmm2160; SSE2-NEXT: pand %xmm1, %xmm2161; SSE2-NEXT: paddd %xmm3, %xmm2162; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]163; SSE2-NEXT: pmuludq %xmm1, %xmm0164; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]165; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]166; SSE2-NEXT: pmuludq %xmm3, %xmm1167; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]168; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]169; SSE2-NEXT: psubd %xmm2, %xmm4170; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]171; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]172; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]173; SSE2-NEXT: movq %xmm2, (%rdi)174; SSE2-NEXT: psrad $31, %xmm2175; SSE2-NEXT: pcmpeqd %xmm4, %xmm2176; SSE2-NEXT: pcmpeqd %xmm1, %xmm1177; SSE2-NEXT: pxor %xmm2, %xmm1178; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]179; SSE2-NEXT: movd %xmm0, 8(%rdi)180; SSE2-NEXT: movdqa %xmm1, %xmm0181; SSE2-NEXT: retq182;183; SSSE3-LABEL: smulo_v3i32:184; SSSE3: # %bb.0:185; SSSE3-NEXT: pxor %xmm2, %xmm2186; SSSE3-NEXT: pxor %xmm3, %xmm3187; SSSE3-NEXT: pcmpgtd %xmm1, %xmm3188; SSSE3-NEXT: pand %xmm0, %xmm3189; SSSE3-NEXT: pcmpgtd %xmm0, %xmm2190; SSSE3-NEXT: pand %xmm1, %xmm2191; SSSE3-NEXT: paddd %xmm3, %xmm2192; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]193; SSSE3-NEXT: pmuludq %xmm1, %xmm0194; SSSE3-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]195; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]196; SSSE3-NEXT: pmuludq %xmm3, %xmm1197; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]198; SSSE3-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]199; SSSE3-NEXT: psubd %xmm2, %xmm4200; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]201; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]202; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]203; SSSE3-NEXT: movq %xmm2, (%rdi)204; SSSE3-NEXT: psrad $31, %xmm2205; SSSE3-NEXT: pcmpeqd %xmm4, %xmm2206; SSSE3-NEXT: pcmpeqd %xmm1, %xmm1207; SSSE3-NEXT: pxor %xmm2, %xmm1208; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]209; SSSE3-NEXT: movd %xmm0, 8(%rdi)210; SSSE3-NEXT: movdqa %xmm1, %xmm0211; SSSE3-NEXT: retq212;213; SSE41-LABEL: smulo_v3i32:214; SSE41: # %bb.0:215; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]216; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]217; SSE41-NEXT: pmuldq %xmm2, %xmm3218; SSE41-NEXT: movdqa %xmm0, %xmm2219; SSE41-NEXT: pmuldq %xmm1, %xmm2220; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]221; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]222; SSE41-NEXT: pmulld %xmm1, %xmm0223; SSE41-NEXT: pextrd $2, %xmm0, 8(%rdi)224; SSE41-NEXT: movq %xmm0, (%rdi)225; SSE41-NEXT: movdqa %xmm0, %xmm1226; SSE41-NEXT: psrad $31, %xmm1227; SSE41-NEXT: pcmpeqd %xmm2, %xmm1228; SSE41-NEXT: pcmpeqd %xmm0, %xmm0229; SSE41-NEXT: pxor %xmm1, %xmm0230; SSE41-NEXT: retq231;232; AVX1-LABEL: smulo_v3i32:233; AVX1: # %bb.0:234; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]235; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]236; AVX1-NEXT: vpmuldq %xmm2, %xmm3, %xmm2237; AVX1-NEXT: vpmuldq %xmm1, %xmm0, %xmm3238; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]239; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]240; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm1241; AVX1-NEXT: vpsrad $31, %xmm1, %xmm0242; AVX1-NEXT: vpcmpeqd %xmm0, %xmm2, %xmm0243; AVX1-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2244; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0245; AVX1-NEXT: vpextrd $2, %xmm1, 8(%rdi)246; AVX1-NEXT: vmovq %xmm1, (%rdi)247; AVX1-NEXT: retq248;249; AVX2-LABEL: smulo_v3i32:250; AVX2: # %bb.0:251; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]252; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]253; AVX2-NEXT: vpmuldq %xmm2, %xmm3, %xmm2254; AVX2-NEXT: vpmuldq %xmm1, %xmm0, %xmm3255; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]256; AVX2-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]257; AVX2-NEXT: vpmulld %xmm1, %xmm0, %xmm1258; AVX2-NEXT: vpsrad $31, %xmm1, %xmm0259; AVX2-NEXT: vpcmpeqd %xmm0, %xmm2, %xmm0260; AVX2-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2261; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm0262; AVX2-NEXT: vpextrd $2, %xmm1, 8(%rdi)263; AVX2-NEXT: vmovq %xmm1, (%rdi)264; AVX2-NEXT: retq265;266; AVX512-LABEL: smulo_v3i32:267; AVX512: # %bb.0:268; AVX512-NEXT: vpmuldq %xmm1, %xmm0, %xmm2269; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]270; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]271; AVX512-NEXT: vpmuldq %xmm3, %xmm4, %xmm3272; AVX512-NEXT: vpmovsxbd {{.*#+}} xmm4 = [1,5,3,7]273; AVX512-NEXT: vpermi2d %xmm3, %xmm2, %xmm4274; AVX512-NEXT: vpmulld %xmm1, %xmm0, %xmm1275; AVX512-NEXT: vpsrad $31, %xmm1, %xmm0276; AVX512-NEXT: vpcmpneqd %xmm0, %xmm4, %k1277; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0278; AVX512-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}279; AVX512-NEXT: vpextrd $2, %xmm1, 8(%rdi)280; AVX512-NEXT: vmovq %xmm1, (%rdi)281; AVX512-NEXT: retq282 %t = call {<3 x i32>, <3 x i1>} @llvm.smul.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)283 %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0284 %obit = extractvalue {<3 x i32>, <3 x i1>} %t, 1285 %res = sext <3 x i1> %obit to <3 x i32>286 store <3 x i32> %val, ptr %p2287 ret <3 x i32> %res288}289 290define <4 x i32> @smulo_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %p2) nounwind {291; SSE2-LABEL: smulo_v4i32:292; SSE2: # %bb.0:293; SSE2-NEXT: pxor %xmm2, %xmm2294; SSE2-NEXT: pxor %xmm3, %xmm3295; SSE2-NEXT: pcmpgtd %xmm1, %xmm3296; SSE2-NEXT: pand %xmm0, %xmm3297; SSE2-NEXT: pcmpgtd %xmm0, %xmm2298; SSE2-NEXT: pand %xmm1, %xmm2299; SSE2-NEXT: paddd %xmm3, %xmm2300; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]301; SSE2-NEXT: pmuludq %xmm1, %xmm0302; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]303; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]304; SSE2-NEXT: pmuludq %xmm3, %xmm1305; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]306; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]307; SSE2-NEXT: psubd %xmm2, %xmm4308; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]309; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]310; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]311; SSE2-NEXT: movdqa %xmm0, (%rdi)312; SSE2-NEXT: movdqa %xmm0, %xmm1313; SSE2-NEXT: psrad $31, %xmm1314; SSE2-NEXT: pcmpeqd %xmm4, %xmm1315; SSE2-NEXT: pcmpeqd %xmm0, %xmm0316; SSE2-NEXT: pxor %xmm1, %xmm0317; SSE2-NEXT: retq318;319; SSSE3-LABEL: smulo_v4i32:320; SSSE3: # %bb.0:321; SSSE3-NEXT: pxor %xmm2, %xmm2322; SSSE3-NEXT: pxor %xmm3, %xmm3323; SSSE3-NEXT: pcmpgtd %xmm1, %xmm3324; SSSE3-NEXT: pand %xmm0, %xmm3325; SSSE3-NEXT: pcmpgtd %xmm0, %xmm2326; SSSE3-NEXT: pand %xmm1, %xmm2327; SSSE3-NEXT: paddd %xmm3, %xmm2328; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]329; SSSE3-NEXT: pmuludq %xmm1, %xmm0330; SSSE3-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]331; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]332; SSSE3-NEXT: pmuludq %xmm3, %xmm1333; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]334; SSSE3-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]335; SSSE3-NEXT: psubd %xmm2, %xmm4336; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]337; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]338; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]339; SSSE3-NEXT: movdqa %xmm0, (%rdi)340; SSSE3-NEXT: movdqa %xmm0, %xmm1341; SSSE3-NEXT: psrad $31, %xmm1342; SSSE3-NEXT: pcmpeqd %xmm4, %xmm1343; SSSE3-NEXT: pcmpeqd %xmm0, %xmm0344; SSSE3-NEXT: pxor %xmm1, %xmm0345; SSSE3-NEXT: retq346;347; SSE41-LABEL: smulo_v4i32:348; SSE41: # %bb.0:349; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]350; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]351; SSE41-NEXT: pmuldq %xmm2, %xmm3352; SSE41-NEXT: movdqa %xmm0, %xmm2353; SSE41-NEXT: pmuldq %xmm1, %xmm2354; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]355; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]356; SSE41-NEXT: pmulld %xmm1, %xmm0357; SSE41-NEXT: movdqa %xmm0, (%rdi)358; SSE41-NEXT: movdqa %xmm0, %xmm1359; SSE41-NEXT: psrad $31, %xmm1360; SSE41-NEXT: pcmpeqd %xmm2, %xmm1361; SSE41-NEXT: pcmpeqd %xmm0, %xmm0362; SSE41-NEXT: pxor %xmm1, %xmm0363; SSE41-NEXT: retq364;365; AVX1-LABEL: smulo_v4i32:366; AVX1: # %bb.0:367; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]368; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]369; AVX1-NEXT: vpmuldq %xmm2, %xmm3, %xmm2370; AVX1-NEXT: vpmuldq %xmm1, %xmm0, %xmm3371; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]372; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]373; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm1374; AVX1-NEXT: vpsrad $31, %xmm1, %xmm0375; AVX1-NEXT: vpcmpeqd %xmm0, %xmm2, %xmm0376; AVX1-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2377; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0378; AVX1-NEXT: vmovdqa %xmm1, (%rdi)379; AVX1-NEXT: retq380;381; AVX2-LABEL: smulo_v4i32:382; AVX2: # %bb.0:383; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]384; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]385; AVX2-NEXT: vpmuldq %xmm2, %xmm3, %xmm2386; AVX2-NEXT: vpmuldq %xmm1, %xmm0, %xmm3387; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]388; AVX2-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]389; AVX2-NEXT: vpmulld %xmm1, %xmm0, %xmm1390; AVX2-NEXT: vpsrad $31, %xmm1, %xmm0391; AVX2-NEXT: vpcmpeqd %xmm0, %xmm2, %xmm0392; AVX2-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2393; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm0394; AVX2-NEXT: vmovdqa %xmm1, (%rdi)395; AVX2-NEXT: retq396;397; AVX512-LABEL: smulo_v4i32:398; AVX512: # %bb.0:399; AVX512-NEXT: vpmuldq %xmm1, %xmm0, %xmm2400; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]401; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]402; AVX512-NEXT: vpmuldq %xmm3, %xmm4, %xmm3403; AVX512-NEXT: vpmovsxbd {{.*#+}} xmm4 = [1,5,3,7]404; AVX512-NEXT: vpermi2d %xmm3, %xmm2, %xmm4405; AVX512-NEXT: vpmulld %xmm1, %xmm0, %xmm1406; AVX512-NEXT: vpsrad $31, %xmm1, %xmm0407; AVX512-NEXT: vpcmpneqd %xmm0, %xmm4, %k1408; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0409; AVX512-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}410; AVX512-NEXT: vmovdqa %xmm1, (%rdi)411; AVX512-NEXT: retq412 %t = call {<4 x i32>, <4 x i1>} @llvm.smul.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a1)413 %val = extractvalue {<4 x i32>, <4 x i1>} %t, 0414 %obit = extractvalue {<4 x i32>, <4 x i1>} %t, 1415 %res = sext <4 x i1> %obit to <4 x i32>416 store <4 x i32> %val, ptr %p2417 ret <4 x i32> %res418}419 420define <6 x i32> @smulo_v6i32(<6 x i32> %a0, <6 x i32> %a1, ptr %p2) nounwind {421; SSE2-LABEL: smulo_v6i32:422; SSE2: # %bb.0:423; SSE2-NEXT: movq %rdi, %rax424; SSE2-NEXT: movd %r8d, %xmm0425; SSE2-NEXT: movd %ecx, %xmm1426; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]427; SSE2-NEXT: movd %edx, %xmm0428; SSE2-NEXT: movd %esi, %xmm4429; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]430; SSE2-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]431; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero432; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero433; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]434; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero435; SSE2-NEXT: movd {{.*#+}} xmm5 = mem[0],zero,zero,zero436; SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]437; SSE2-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]438; SSE2-NEXT: movd %r9d, %xmm0439; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero440; SSE2-NEXT: movdqa %xmm0, %xmm3441; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]442; SSE2-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero443; SSE2-NEXT: movd {{.*#+}} xmm7 = mem[0],zero,zero,zero444; SSE2-NEXT: pmuludq %xmm7, %xmm0445; SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]446; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rcx447; SSE2-NEXT: pxor %xmm6, %xmm6448; SSE2-NEXT: pxor %xmm8, %xmm8449; SSE2-NEXT: pcmpgtd %xmm5, %xmm8450; SSE2-NEXT: pand %xmm4, %xmm8451; SSE2-NEXT: pxor %xmm9, %xmm9452; SSE2-NEXT: pcmpgtd %xmm4, %xmm9453; SSE2-NEXT: pand %xmm5, %xmm9454; SSE2-NEXT: paddd %xmm8, %xmm9455; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm4[1,1,3,3]456; SSE2-NEXT: pmuludq %xmm5, %xmm4457; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm4[1,3,2,3]458; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]459; SSE2-NEXT: pmuludq %xmm8, %xmm5460; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm5[1,3,2,3]461; SSE2-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm8[0],xmm10[1],xmm8[1]462; SSE2-NEXT: psubd %xmm9, %xmm10463; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]464; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]465; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]466; SSE2-NEXT: movdqa %xmm4, (%rcx)467; SSE2-NEXT: psrad $31, %xmm4468; SSE2-NEXT: pcmpeqd %xmm10, %xmm4469; SSE2-NEXT: pcmpeqd %xmm5, %xmm5470; SSE2-NEXT: pxor %xmm5, %xmm4471; SSE2-NEXT: pxor %xmm8, %xmm8472; SSE2-NEXT: pcmpgtd %xmm7, %xmm8473; SSE2-NEXT: pand %xmm3, %xmm8474; SSE2-NEXT: pcmpgtd %xmm3, %xmm6475; SSE2-NEXT: pand %xmm7, %xmm6476; SSE2-NEXT: paddd %xmm8, %xmm6477; SSE2-NEXT: pmuludq %xmm2, %xmm1478; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]479; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]480; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]481; SSE2-NEXT: psubd %xmm6, %xmm3482; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]483; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]484; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]485; SSE2-NEXT: movq %xmm0, 16(%rcx)486; SSE2-NEXT: psrad $31, %xmm0487; SSE2-NEXT: pcmpeqd %xmm3, %xmm0488; SSE2-NEXT: pxor %xmm5, %xmm0489; SSE2-NEXT: movq %xmm0, 16(%rdi)490; SSE2-NEXT: movdqa %xmm4, (%rdi)491; SSE2-NEXT: retq492;493; SSSE3-LABEL: smulo_v6i32:494; SSSE3: # %bb.0:495; SSSE3-NEXT: movq %rdi, %rax496; SSSE3-NEXT: movd %r8d, %xmm0497; SSSE3-NEXT: movd %ecx, %xmm1498; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]499; SSSE3-NEXT: movd %edx, %xmm0500; SSSE3-NEXT: movd %esi, %xmm4501; SSSE3-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]502; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]503; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero504; SSSE3-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero505; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]506; SSSE3-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero507; SSSE3-NEXT: movd {{.*#+}} xmm5 = mem[0],zero,zero,zero508; SSSE3-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]509; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]510; SSSE3-NEXT: movd %r9d, %xmm0511; SSSE3-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero512; SSSE3-NEXT: movdqa %xmm0, %xmm3513; SSSE3-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]514; SSSE3-NEXT: movd {{.*#+}} xmm2 = mem[0],zero,zero,zero515; SSSE3-NEXT: movd {{.*#+}} xmm7 = mem[0],zero,zero,zero516; SSSE3-NEXT: pmuludq %xmm7, %xmm0517; SSSE3-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]518; SSSE3-NEXT: movq {{[0-9]+}}(%rsp), %rcx519; SSSE3-NEXT: pxor %xmm6, %xmm6520; SSSE3-NEXT: pxor %xmm8, %xmm8521; SSSE3-NEXT: pcmpgtd %xmm5, %xmm8522; SSSE3-NEXT: pand %xmm4, %xmm8523; SSSE3-NEXT: pxor %xmm9, %xmm9524; SSSE3-NEXT: pcmpgtd %xmm4, %xmm9525; SSSE3-NEXT: pand %xmm5, %xmm9526; SSSE3-NEXT: paddd %xmm8, %xmm9527; SSSE3-NEXT: pshufd {{.*#+}} xmm8 = xmm4[1,1,3,3]528; SSSE3-NEXT: pmuludq %xmm5, %xmm4529; SSSE3-NEXT: pshufd {{.*#+}} xmm10 = xmm4[1,3,2,3]530; SSSE3-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]531; SSSE3-NEXT: pmuludq %xmm8, %xmm5532; SSSE3-NEXT: pshufd {{.*#+}} xmm8 = xmm5[1,3,2,3]533; SSSE3-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm8[0],xmm10[1],xmm8[1]534; SSSE3-NEXT: psubd %xmm9, %xmm10535; SSSE3-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]536; SSSE3-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]537; SSSE3-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]538; SSSE3-NEXT: movdqa %xmm4, (%rcx)539; SSSE3-NEXT: psrad $31, %xmm4540; SSSE3-NEXT: pcmpeqd %xmm10, %xmm4541; SSSE3-NEXT: pcmpeqd %xmm5, %xmm5542; SSSE3-NEXT: pxor %xmm5, %xmm4543; SSSE3-NEXT: pxor %xmm8, %xmm8544; SSSE3-NEXT: pcmpgtd %xmm7, %xmm8545; SSSE3-NEXT: pand %xmm3, %xmm8546; SSSE3-NEXT: pcmpgtd %xmm3, %xmm6547; SSSE3-NEXT: pand %xmm7, %xmm6548; SSSE3-NEXT: paddd %xmm8, %xmm6549; SSSE3-NEXT: pmuludq %xmm2, %xmm1550; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]551; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]552; SSSE3-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]553; SSSE3-NEXT: psubd %xmm6, %xmm3554; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]555; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]556; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]557; SSSE3-NEXT: movq %xmm0, 16(%rcx)558; SSSE3-NEXT: psrad $31, %xmm0559; SSSE3-NEXT: pcmpeqd %xmm3, %xmm0560; SSSE3-NEXT: pxor %xmm5, %xmm0561; SSSE3-NEXT: movq %xmm0, 16(%rdi)562; SSSE3-NEXT: movdqa %xmm4, (%rdi)563; SSSE3-NEXT: retq564;565; SSE41-LABEL: smulo_v6i32:566; SSE41: # %bb.0:567; SSE41-NEXT: movq %rdi, %rax568; SSE41-NEXT: movd %esi, %xmm2569; SSE41-NEXT: pinsrd $1, %edx, %xmm2570; SSE41-NEXT: pinsrd $2, %ecx, %xmm2571; SSE41-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero572; SSE41-NEXT: pinsrd $1, {{[0-9]+}}(%rsp), %xmm0573; SSE41-NEXT: pinsrd $2, {{[0-9]+}}(%rsp), %xmm0574; SSE41-NEXT: movdqa %xmm0, %xmm1575; SSE41-NEXT: pmuldq %xmm2, %xmm0576; SSE41-NEXT: pinsrd $3, %r8d, %xmm2577; SSE41-NEXT: movl {{[0-9]+}}(%rsp), %ecx578; SSE41-NEXT: movd {{.*#+}} xmm3 = mem[0],zero,zero,zero579; SSE41-NEXT: movd %r9d, %xmm4580; SSE41-NEXT: movdqa %xmm4, %xmm5581; SSE41-NEXT: pmuldq %xmm3, %xmm4582; SSE41-NEXT: pinsrd $1, %ecx, %xmm3583; SSE41-NEXT: movl {{[0-9]+}}(%rsp), %edx584; SSE41-NEXT: pinsrd $1, %edx, %xmm5585; SSE41-NEXT: pmulld %xmm3, %xmm5586; SSE41-NEXT: pinsrd $3, {{[0-9]+}}(%rsp), %xmm1587; SSE41-NEXT: movq {{[0-9]+}}(%rsp), %rsi588; SSE41-NEXT: movd %ecx, %xmm3589; SSE41-NEXT: movd %edx, %xmm6590; SSE41-NEXT: pmuldq %xmm3, %xmm6591; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]592; SSE41-NEXT: pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm6[2,3],xmm3[4,5],xmm6[6,7]593; SSE41-NEXT: movq %xmm5, 16(%rsi)594; SSE41-NEXT: psrad $31, %xmm5595; SSE41-NEXT: pcmpeqd %xmm3, %xmm5596; SSE41-NEXT: pcmpeqd %xmm3, %xmm3597; SSE41-NEXT: pxor %xmm3, %xmm5598; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]599; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]600; SSE41-NEXT: pmuldq %xmm4, %xmm6601; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]602; SSE41-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm6[2,3],xmm0[4,5],xmm6[6,7]603; SSE41-NEXT: pmulld %xmm2, %xmm1604; SSE41-NEXT: movdqa %xmm1, (%rsi)605; SSE41-NEXT: psrad $31, %xmm1606; SSE41-NEXT: pcmpeqd %xmm0, %xmm1607; SSE41-NEXT: pxor %xmm3, %xmm1608; SSE41-NEXT: movq %xmm5, 16(%rdi)609; SSE41-NEXT: movdqa %xmm1, (%rdi)610; SSE41-NEXT: retq611;612; AVX1-LABEL: smulo_v6i32:613; AVX1: # %bb.0:614; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2615; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]616; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4617; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]618; AVX1-NEXT: vpmuldq %xmm3, %xmm5, %xmm3619; AVX1-NEXT: vpmuldq %xmm2, %xmm4, %xmm5620; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]621; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3],xmm5[4,5],xmm3[6,7]622; AVX1-NEXT: vpmulld %xmm2, %xmm4, %xmm2623; AVX1-NEXT: vpsrad $31, %xmm2, %xmm4624; AVX1-NEXT: vpcmpeqd %xmm4, %xmm3, %xmm3625; AVX1-NEXT: vpcmpeqd %xmm4, %xmm4, %xmm4626; AVX1-NEXT: vpxor %xmm4, %xmm3, %xmm3627; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]628; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]629; AVX1-NEXT: vpmuldq %xmm5, %xmm6, %xmm5630; AVX1-NEXT: vpmuldq %xmm1, %xmm0, %xmm6631; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]632; AVX1-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5],xmm5[6,7]633; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm1634; AVX1-NEXT: vpsrad $31, %xmm1, %xmm0635; AVX1-NEXT: vpcmpeqd %xmm0, %xmm5, %xmm0636; AVX1-NEXT: vpxor %xmm4, %xmm0, %xmm0637; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0638; AVX1-NEXT: vmovq %xmm2, 16(%rdi)639; AVX1-NEXT: vmovdqa %xmm1, (%rdi)640; AVX1-NEXT: retq641;642; AVX2-LABEL: smulo_v6i32:643; AVX2: # %bb.0:644; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]645; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]646; AVX2-NEXT: vpmuldq %ymm2, %ymm3, %ymm2647; AVX2-NEXT: vpmuldq %ymm1, %ymm0, %ymm3648; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]649; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]650; AVX2-NEXT: vpmulld %ymm1, %ymm0, %ymm1651; AVX2-NEXT: vpsrad $31, %ymm1, %ymm0652; AVX2-NEXT: vpcmpeqd %ymm0, %ymm2, %ymm0653; AVX2-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2654; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm0655; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2656; AVX2-NEXT: vmovq %xmm2, 16(%rdi)657; AVX2-NEXT: vmovdqa %xmm1, (%rdi)658; AVX2-NEXT: retq659;660; AVX512-LABEL: smulo_v6i32:661; AVX512: # %bb.0:662; AVX512-NEXT: vpmuldq %ymm1, %ymm0, %ymm2663; AVX512-NEXT: vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]664; AVX512-NEXT: vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]665; AVX512-NEXT: vpmuldq %ymm3, %ymm4, %ymm3666; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]667; AVX512-NEXT: vpermi2d %ymm3, %ymm2, %ymm4668; AVX512-NEXT: vpmulld %ymm1, %ymm0, %ymm1669; AVX512-NEXT: vpsrad $31, %ymm1, %ymm0670; AVX512-NEXT: vpcmpneqd %ymm0, %ymm4, %k1671; AVX512-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0672; AVX512-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z}673; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm2674; AVX512-NEXT: vmovq %xmm2, 16(%rdi)675; AVX512-NEXT: vmovdqa %xmm1, (%rdi)676; AVX512-NEXT: retq677 %t = call {<6 x i32>, <6 x i1>} @llvm.smul.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1)678 %val = extractvalue {<6 x i32>, <6 x i1>} %t, 0679 %obit = extractvalue {<6 x i32>, <6 x i1>} %t, 1680 %res = sext <6 x i1> %obit to <6 x i32>681 store <6 x i32> %val, ptr %p2682 ret <6 x i32> %res683}684 685define <8 x i32> @smulo_v8i32(<8 x i32> %a0, <8 x i32> %a1, ptr %p2) nounwind {686; SSE2-LABEL: smulo_v8i32:687; SSE2: # %bb.0:688; SSE2-NEXT: pxor %xmm4, %xmm4689; SSE2-NEXT: pxor %xmm5, %xmm5690; SSE2-NEXT: pcmpgtd %xmm2, %xmm5691; SSE2-NEXT: pand %xmm0, %xmm5692; SSE2-NEXT: pxor %xmm6, %xmm6693; SSE2-NEXT: pcmpgtd %xmm0, %xmm6694; SSE2-NEXT: pand %xmm2, %xmm6695; SSE2-NEXT: paddd %xmm5, %xmm6696; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]697; SSE2-NEXT: pmuludq %xmm2, %xmm0698; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]699; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]700; SSE2-NEXT: pmuludq %xmm7, %xmm2701; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm2[1,3,2,3]702; SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]703; SSE2-NEXT: psubd %xmm6, %xmm5704; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]705; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]706; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]707; SSE2-NEXT: movdqa %xmm0, (%rdi)708; SSE2-NEXT: psrad $31, %xmm0709; SSE2-NEXT: pcmpeqd %xmm5, %xmm0710; SSE2-NEXT: pcmpeqd %xmm2, %xmm2711; SSE2-NEXT: pxor %xmm2, %xmm0712; SSE2-NEXT: pxor %xmm5, %xmm5713; SSE2-NEXT: pcmpgtd %xmm3, %xmm5714; SSE2-NEXT: pand %xmm1, %xmm5715; SSE2-NEXT: pcmpgtd %xmm1, %xmm4716; SSE2-NEXT: pand %xmm3, %xmm4717; SSE2-NEXT: paddd %xmm5, %xmm4718; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]719; SSE2-NEXT: pmuludq %xmm3, %xmm1720; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm1[1,3,2,3]721; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]722; SSE2-NEXT: pmuludq %xmm5, %xmm3723; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]724; SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]725; SSE2-NEXT: psubd %xmm4, %xmm6726; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]727; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]728; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]729; SSE2-NEXT: movdqa %xmm1, 16(%rdi)730; SSE2-NEXT: psrad $31, %xmm1731; SSE2-NEXT: pcmpeqd %xmm6, %xmm1732; SSE2-NEXT: pxor %xmm2, %xmm1733; SSE2-NEXT: retq734;735; SSSE3-LABEL: smulo_v8i32:736; SSSE3: # %bb.0:737; SSSE3-NEXT: pxor %xmm4, %xmm4738; SSSE3-NEXT: pxor %xmm5, %xmm5739; SSSE3-NEXT: pcmpgtd %xmm2, %xmm5740; SSSE3-NEXT: pand %xmm0, %xmm5741; SSSE3-NEXT: pxor %xmm6, %xmm6742; SSSE3-NEXT: pcmpgtd %xmm0, %xmm6743; SSSE3-NEXT: pand %xmm2, %xmm6744; SSSE3-NEXT: paddd %xmm5, %xmm6745; SSSE3-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]746; SSSE3-NEXT: pmuludq %xmm2, %xmm0747; SSSE3-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]748; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]749; SSSE3-NEXT: pmuludq %xmm7, %xmm2750; SSSE3-NEXT: pshufd {{.*#+}} xmm7 = xmm2[1,3,2,3]751; SSSE3-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]752; SSSE3-NEXT: psubd %xmm6, %xmm5753; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]754; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]755; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]756; SSSE3-NEXT: movdqa %xmm0, (%rdi)757; SSSE3-NEXT: psrad $31, %xmm0758; SSSE3-NEXT: pcmpeqd %xmm5, %xmm0759; SSSE3-NEXT: pcmpeqd %xmm2, %xmm2760; SSSE3-NEXT: pxor %xmm2, %xmm0761; SSSE3-NEXT: pxor %xmm5, %xmm5762; SSSE3-NEXT: pcmpgtd %xmm3, %xmm5763; SSSE3-NEXT: pand %xmm1, %xmm5764; SSSE3-NEXT: pcmpgtd %xmm1, %xmm4765; SSSE3-NEXT: pand %xmm3, %xmm4766; SSSE3-NEXT: paddd %xmm5, %xmm4767; SSSE3-NEXT: pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]768; SSSE3-NEXT: pmuludq %xmm3, %xmm1769; SSSE3-NEXT: pshufd {{.*#+}} xmm6 = xmm1[1,3,2,3]770; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]771; SSSE3-NEXT: pmuludq %xmm5, %xmm3772; SSSE3-NEXT: pshufd {{.*#+}} xmm5 = xmm3[1,3,2,3]773; SSSE3-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]774; SSSE3-NEXT: psubd %xmm4, %xmm6775; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]776; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]777; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]778; SSSE3-NEXT: movdqa %xmm1, 16(%rdi)779; SSSE3-NEXT: psrad $31, %xmm1780; SSSE3-NEXT: pcmpeqd %xmm6, %xmm1781; SSSE3-NEXT: pxor %xmm2, %xmm1782; SSSE3-NEXT: retq783;784; SSE41-LABEL: smulo_v8i32:785; SSE41: # %bb.0:786; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]787; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]788; SSE41-NEXT: pmuldq %xmm4, %xmm5789; SSE41-NEXT: movdqa %xmm0, %xmm4790; SSE41-NEXT: pmuldq %xmm2, %xmm4791; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]792; SSE41-NEXT: pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]793; SSE41-NEXT: pmulld %xmm2, %xmm0794; SSE41-NEXT: movdqa %xmm0, (%rdi)795; SSE41-NEXT: psrad $31, %xmm0796; SSE41-NEXT: pcmpeqd %xmm4, %xmm0797; SSE41-NEXT: pcmpeqd %xmm2, %xmm2798; SSE41-NEXT: pxor %xmm2, %xmm0799; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]800; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]801; SSE41-NEXT: pmuldq %xmm4, %xmm5802; SSE41-NEXT: movdqa %xmm1, %xmm4803; SSE41-NEXT: pmuldq %xmm3, %xmm4804; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]805; SSE41-NEXT: pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]806; SSE41-NEXT: pmulld %xmm3, %xmm1807; SSE41-NEXT: movdqa %xmm1, 16(%rdi)808; SSE41-NEXT: psrad $31, %xmm1809; SSE41-NEXT: pcmpeqd %xmm4, %xmm1810; SSE41-NEXT: pxor %xmm2, %xmm1811; SSE41-NEXT: retq812;813; AVX1-LABEL: smulo_v8i32:814; AVX1: # %bb.0:815; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2816; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]817; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4818; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]819; AVX1-NEXT: vpmuldq %xmm3, %xmm5, %xmm3820; AVX1-NEXT: vpmuldq %xmm2, %xmm4, %xmm5821; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]822; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3],xmm5[4,5],xmm3[6,7]823; AVX1-NEXT: vpmulld %xmm2, %xmm4, %xmm2824; AVX1-NEXT: vpsrad $31, %xmm2, %xmm4825; AVX1-NEXT: vpcmpeqd %xmm4, %xmm3, %xmm3826; AVX1-NEXT: vpcmpeqd %xmm4, %xmm4, %xmm4827; AVX1-NEXT: vpxor %xmm4, %xmm3, %xmm3828; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]829; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]830; AVX1-NEXT: vpmuldq %xmm5, %xmm6, %xmm5831; AVX1-NEXT: vpmuldq %xmm1, %xmm0, %xmm6832; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]833; AVX1-NEXT: vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5],xmm5[6,7]834; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm1835; AVX1-NEXT: vpsrad $31, %xmm1, %xmm0836; AVX1-NEXT: vpcmpeqd %xmm0, %xmm5, %xmm0837; AVX1-NEXT: vpxor %xmm4, %xmm0, %xmm0838; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm0, %ymm0839; AVX1-NEXT: vmovdqa %xmm2, 16(%rdi)840; AVX1-NEXT: vmovdqa %xmm1, (%rdi)841; AVX1-NEXT: retq842;843; AVX2-LABEL: smulo_v8i32:844; AVX2: # %bb.0:845; AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]846; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm0[1,1,3,3,5,5,7,7]847; AVX2-NEXT: vpmuldq %ymm2, %ymm3, %ymm2848; AVX2-NEXT: vpmuldq %ymm1, %ymm0, %ymm3849; AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[1,1,3,3,5,5,7,7]850; AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4],ymm2[5],ymm3[6],ymm2[7]851; AVX2-NEXT: vpmulld %ymm1, %ymm0, %ymm1852; AVX2-NEXT: vpsrad $31, %ymm1, %ymm0853; AVX2-NEXT: vpcmpeqd %ymm0, %ymm2, %ymm0854; AVX2-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2855; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm0856; AVX2-NEXT: vmovdqa %ymm1, (%rdi)857; AVX2-NEXT: retq858;859; AVX512-LABEL: smulo_v8i32:860; AVX512: # %bb.0:861; AVX512-NEXT: vpmuldq %ymm1, %ymm0, %ymm2862; AVX512-NEXT: vpshufd {{.*#+}} ymm3 = ymm1[1,1,3,3,5,5,7,7]863; AVX512-NEXT: vpshufd {{.*#+}} ymm4 = ymm0[1,1,3,3,5,5,7,7]864; AVX512-NEXT: vpmuldq %ymm3, %ymm4, %ymm3865; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm4 = [1,9,3,11,5,13,7,15]866; AVX512-NEXT: vpermi2d %ymm3, %ymm2, %ymm4867; AVX512-NEXT: vpmulld %ymm1, %ymm0, %ymm1868; AVX512-NEXT: vpsrad $31, %ymm1, %ymm0869; AVX512-NEXT: vpcmpneqd %ymm0, %ymm4, %k1870; AVX512-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0871; AVX512-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z}872; AVX512-NEXT: vmovdqa %ymm1, (%rdi)873; AVX512-NEXT: retq874 %t = call {<8 x i32>, <8 x i1>} @llvm.smul.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1)875 %val = extractvalue {<8 x i32>, <8 x i1>} %t, 0876 %obit = extractvalue {<8 x i32>, <8 x i1>} %t, 1877 %res = sext <8 x i1> %obit to <8 x i32>878 store <8 x i32> %val, ptr %p2879 ret <8 x i32> %res880}881 882define <16 x i32> @smulo_v16i32(<16 x i32> %a0, <16 x i32> %a1, ptr %p2) nounwind {883; SSE2-LABEL: smulo_v16i32:884; SSE2: # %bb.0:885; SSE2-NEXT: pxor %xmm8, %xmm8886; SSE2-NEXT: pxor %xmm9, %xmm9887; SSE2-NEXT: pcmpgtd %xmm4, %xmm9888; SSE2-NEXT: pand %xmm0, %xmm9889; SSE2-NEXT: pxor %xmm10, %xmm10890; SSE2-NEXT: pcmpgtd %xmm0, %xmm10891; SSE2-NEXT: pand %xmm4, %xmm10892; SSE2-NEXT: paddd %xmm9, %xmm10893; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3]894; SSE2-NEXT: pmuludq %xmm4, %xmm0895; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm0[1,3,2,3]896; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]897; SSE2-NEXT: pmuludq %xmm11, %xmm4898; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm4[1,3,2,3]899; SSE2-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]900; SSE2-NEXT: psubd %xmm10, %xmm9901; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]902; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]903; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]904; SSE2-NEXT: movdqa %xmm0, (%rdi)905; SSE2-NEXT: psrad $31, %xmm0906; SSE2-NEXT: pcmpeqd %xmm9, %xmm0907; SSE2-NEXT: pcmpeqd %xmm4, %xmm4908; SSE2-NEXT: pxor %xmm4, %xmm0909; SSE2-NEXT: pxor %xmm9, %xmm9910; SSE2-NEXT: pcmpgtd %xmm5, %xmm9911; SSE2-NEXT: pand %xmm1, %xmm9912; SSE2-NEXT: pxor %xmm10, %xmm10913; SSE2-NEXT: pcmpgtd %xmm1, %xmm10914; SSE2-NEXT: pand %xmm5, %xmm10915; SSE2-NEXT: paddd %xmm9, %xmm10916; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm1[1,1,3,3]917; SSE2-NEXT: pmuludq %xmm5, %xmm1918; SSE2-NEXT: pshufd {{.*#+}} xmm11 = xmm1[1,3,2,3]919; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]920; SSE2-NEXT: pmuludq %xmm9, %xmm5921; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm5[1,3,2,3]922; SSE2-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]923; SSE2-NEXT: psubd %xmm10, %xmm11924; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]925; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]926; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]927; SSE2-NEXT: movdqa %xmm1, 16(%rdi)928; SSE2-NEXT: psrad $31, %xmm1929; SSE2-NEXT: pcmpeqd %xmm11, %xmm1930; SSE2-NEXT: pxor %xmm4, %xmm1931; SSE2-NEXT: pxor %xmm5, %xmm5932; SSE2-NEXT: pcmpgtd %xmm6, %xmm5933; SSE2-NEXT: pand %xmm2, %xmm5934; SSE2-NEXT: pxor %xmm9, %xmm9935; SSE2-NEXT: pcmpgtd %xmm2, %xmm9936; SSE2-NEXT: pand %xmm6, %xmm9937; SSE2-NEXT: paddd %xmm5, %xmm9938; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]939; SSE2-NEXT: pmuludq %xmm6, %xmm2940; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm2[1,3,2,3]941; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]942; SSE2-NEXT: pmuludq %xmm5, %xmm6943; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm6[1,3,2,3]944; SSE2-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm5[0],xmm10[1],xmm5[1]945; SSE2-NEXT: psubd %xmm9, %xmm10946; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]947; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]948; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]949; SSE2-NEXT: movdqa %xmm2, 32(%rdi)950; SSE2-NEXT: psrad $31, %xmm2951; SSE2-NEXT: pcmpeqd %xmm10, %xmm2952; SSE2-NEXT: pxor %xmm4, %xmm2953; SSE2-NEXT: pxor %xmm5, %xmm5954; SSE2-NEXT: pcmpgtd %xmm7, %xmm5955; SSE2-NEXT: pand %xmm3, %xmm5956; SSE2-NEXT: pcmpgtd %xmm3, %xmm8957; SSE2-NEXT: pand %xmm7, %xmm8958; SSE2-NEXT: paddd %xmm5, %xmm8959; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]960; SSE2-NEXT: pmuludq %xmm7, %xmm3961; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm3[1,3,2,3]962; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]963; SSE2-NEXT: pmuludq %xmm5, %xmm7964; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,3,2,3]965; SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]966; SSE2-NEXT: psubd %xmm8, %xmm6967; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]968; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm7[0,2,2,3]969; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]970; SSE2-NEXT: movdqa %xmm3, 48(%rdi)971; SSE2-NEXT: psrad $31, %xmm3972; SSE2-NEXT: pcmpeqd %xmm6, %xmm3973; SSE2-NEXT: pxor %xmm4, %xmm3974; SSE2-NEXT: retq975;976; SSSE3-LABEL: smulo_v16i32:977; SSSE3: # %bb.0:978; SSSE3-NEXT: pxor %xmm8, %xmm8979; SSSE3-NEXT: pxor %xmm9, %xmm9980; SSSE3-NEXT: pcmpgtd %xmm4, %xmm9981; SSSE3-NEXT: pand %xmm0, %xmm9982; SSSE3-NEXT: pxor %xmm10, %xmm10983; SSSE3-NEXT: pcmpgtd %xmm0, %xmm10984; SSSE3-NEXT: pand %xmm4, %xmm10985; SSSE3-NEXT: paddd %xmm9, %xmm10986; SSSE3-NEXT: pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3]987; SSSE3-NEXT: pmuludq %xmm4, %xmm0988; SSSE3-NEXT: pshufd {{.*#+}} xmm9 = xmm0[1,3,2,3]989; SSSE3-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]990; SSSE3-NEXT: pmuludq %xmm11, %xmm4991; SSSE3-NEXT: pshufd {{.*#+}} xmm11 = xmm4[1,3,2,3]992; SSSE3-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm11[0],xmm9[1],xmm11[1]993; SSSE3-NEXT: psubd %xmm10, %xmm9994; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]995; SSSE3-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]996; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]997; SSSE3-NEXT: movdqa %xmm0, (%rdi)998; SSSE3-NEXT: psrad $31, %xmm0999; SSSE3-NEXT: pcmpeqd %xmm9, %xmm01000; SSSE3-NEXT: pcmpeqd %xmm4, %xmm41001; SSSE3-NEXT: pxor %xmm4, %xmm01002; SSSE3-NEXT: pxor %xmm9, %xmm91003; SSSE3-NEXT: pcmpgtd %xmm5, %xmm91004; SSSE3-NEXT: pand %xmm1, %xmm91005; SSSE3-NEXT: pxor %xmm10, %xmm101006; SSSE3-NEXT: pcmpgtd %xmm1, %xmm101007; SSSE3-NEXT: pand %xmm5, %xmm101008; SSSE3-NEXT: paddd %xmm9, %xmm101009; SSSE3-NEXT: pshufd {{.*#+}} xmm9 = xmm1[1,1,3,3]1010; SSSE3-NEXT: pmuludq %xmm5, %xmm11011; SSSE3-NEXT: pshufd {{.*#+}} xmm11 = xmm1[1,3,2,3]1012; SSSE3-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]1013; SSSE3-NEXT: pmuludq %xmm9, %xmm51014; SSSE3-NEXT: pshufd {{.*#+}} xmm9 = xmm5[1,3,2,3]1015; SSSE3-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]1016; SSSE3-NEXT: psubd %xmm10, %xmm111017; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]1018; SSSE3-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]1019; SSSE3-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]1020; SSSE3-NEXT: movdqa %xmm1, 16(%rdi)1021; SSSE3-NEXT: psrad $31, %xmm11022; SSSE3-NEXT: pcmpeqd %xmm11, %xmm11023; SSSE3-NEXT: pxor %xmm4, %xmm11024; SSSE3-NEXT: pxor %xmm5, %xmm51025; SSSE3-NEXT: pcmpgtd %xmm6, %xmm51026; SSSE3-NEXT: pand %xmm2, %xmm51027; SSSE3-NEXT: pxor %xmm9, %xmm91028; SSSE3-NEXT: pcmpgtd %xmm2, %xmm91029; SSSE3-NEXT: pand %xmm6, %xmm91030; SSSE3-NEXT: paddd %xmm5, %xmm91031; SSSE3-NEXT: pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]1032; SSSE3-NEXT: pmuludq %xmm6, %xmm21033; SSSE3-NEXT: pshufd {{.*#+}} xmm10 = xmm2[1,3,2,3]1034; SSSE3-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]1035; SSSE3-NEXT: pmuludq %xmm5, %xmm61036; SSSE3-NEXT: pshufd {{.*#+}} xmm5 = xmm6[1,3,2,3]1037; SSSE3-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm5[0],xmm10[1],xmm5[1]1038; SSSE3-NEXT: psubd %xmm9, %xmm101039; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]1040; SSSE3-NEXT: pshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]1041; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]1042; SSSE3-NEXT: movdqa %xmm2, 32(%rdi)1043; SSSE3-NEXT: psrad $31, %xmm21044; SSSE3-NEXT: pcmpeqd %xmm10, %xmm21045; SSSE3-NEXT: pxor %xmm4, %xmm21046; SSSE3-NEXT: pxor %xmm5, %xmm51047; SSSE3-NEXT: pcmpgtd %xmm7, %xmm51048; SSSE3-NEXT: pand %xmm3, %xmm51049; SSSE3-NEXT: pcmpgtd %xmm3, %xmm81050; SSSE3-NEXT: pand %xmm7, %xmm81051; SSSE3-NEXT: paddd %xmm5, %xmm81052; SSSE3-NEXT: pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]1053; SSSE3-NEXT: pmuludq %xmm7, %xmm31054; SSSE3-NEXT: pshufd {{.*#+}} xmm6 = xmm3[1,3,2,3]1055; SSSE3-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]1056; SSSE3-NEXT: pmuludq %xmm5, %xmm71057; SSSE3-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,3,2,3]1058; SSSE3-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]1059; SSSE3-NEXT: psubd %xmm8, %xmm61060; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]1061; SSSE3-NEXT: pshufd {{.*#+}} xmm5 = xmm7[0,2,2,3]1062; SSSE3-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]1063; SSSE3-NEXT: movdqa %xmm3, 48(%rdi)1064; SSSE3-NEXT: psrad $31, %xmm31065; SSSE3-NEXT: pcmpeqd %xmm6, %xmm31066; SSSE3-NEXT: pxor %xmm4, %xmm31067; SSSE3-NEXT: retq1068;1069; SSE41-LABEL: smulo_v16i32:1070; SSE41: # %bb.0:1071; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm4[1,1,3,3]1072; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[1,1,3,3]1073; SSE41-NEXT: pmuldq %xmm8, %xmm91074; SSE41-NEXT: movdqa %xmm0, %xmm81075; SSE41-NEXT: pmuldq %xmm4, %xmm81076; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]1077; SSE41-NEXT: pblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5],xmm9[6,7]1078; SSE41-NEXT: pmulld %xmm4, %xmm01079; SSE41-NEXT: movdqa %xmm0, (%rdi)1080; SSE41-NEXT: psrad $31, %xmm01081; SSE41-NEXT: pcmpeqd %xmm8, %xmm01082; SSE41-NEXT: pcmpeqd %xmm4, %xmm41083; SSE41-NEXT: pxor %xmm4, %xmm01084; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm5[1,1,3,3]1085; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm1[1,1,3,3]1086; SSE41-NEXT: pmuldq %xmm8, %xmm91087; SSE41-NEXT: movdqa %xmm1, %xmm81088; SSE41-NEXT: pmuldq %xmm5, %xmm81089; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]1090; SSE41-NEXT: pblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2,3],xmm8[4,5],xmm9[6,7]1091; SSE41-NEXT: pmulld %xmm5, %xmm11092; SSE41-NEXT: movdqa %xmm1, 16(%rdi)1093; SSE41-NEXT: psrad $31, %xmm11094; SSE41-NEXT: pcmpeqd %xmm8, %xmm11095; SSE41-NEXT: pxor %xmm4, %xmm11096; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]1097; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm2[1,1,3,3]1098; SSE41-NEXT: pmuldq %xmm5, %xmm81099; SSE41-NEXT: movdqa %xmm2, %xmm51100; SSE41-NEXT: pmuldq %xmm6, %xmm51101; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]1102; SSE41-NEXT: pblendw {{.*#+}} xmm5 = xmm5[0,1],xmm8[2,3],xmm5[4,5],xmm8[6,7]1103; SSE41-NEXT: pmulld %xmm6, %xmm21104; SSE41-NEXT: movdqa %xmm2, 32(%rdi)1105; SSE41-NEXT: psrad $31, %xmm21106; SSE41-NEXT: pcmpeqd %xmm5, %xmm21107; SSE41-NEXT: pxor %xmm4, %xmm21108; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]1109; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm3[1,1,3,3]1110; SSE41-NEXT: pmuldq %xmm5, %xmm61111; SSE41-NEXT: movdqa %xmm3, %xmm51112; SSE41-NEXT: pmuldq %xmm7, %xmm51113; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]1114; SSE41-NEXT: pblendw {{.*#+}} xmm5 = xmm5[0,1],xmm6[2,3],xmm5[4,5],xmm6[6,7]1115; SSE41-NEXT: pmulld %xmm7, %xmm31116; SSE41-NEXT: movdqa %xmm3, 48(%rdi)1117; SSE41-NEXT: psrad $31, %xmm31118; SSE41-NEXT: pcmpeqd %xmm5, %xmm31119; SSE41-NEXT: pxor %xmm4, %xmm31120; SSE41-NEXT: retq1121;1122; AVX1-LABEL: smulo_v16i32:1123; AVX1: # %bb.0:1124; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm41125; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]1126; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm61127; AVX1-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]1128; AVX1-NEXT: vpmuldq %xmm5, %xmm7, %xmm51129; AVX1-NEXT: vpmuldq %xmm4, %xmm6, %xmm71130; AVX1-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]1131; AVX1-NEXT: vpblendw {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3],xmm7[4,5],xmm5[6,7]1132; AVX1-NEXT: vpmulld %xmm4, %xmm6, %xmm41133; AVX1-NEXT: vpsrad $31, %xmm4, %xmm61134; AVX1-NEXT: vpcmpeqd %xmm6, %xmm5, %xmm51135; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,3,3]1136; AVX1-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]1137; AVX1-NEXT: vpmuldq %xmm6, %xmm7, %xmm61138; AVX1-NEXT: vpmuldq %xmm3, %xmm1, %xmm71139; AVX1-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]1140; AVX1-NEXT: vpblendw {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3],xmm7[4,5],xmm6[6,7]1141; AVX1-NEXT: vpmulld %xmm3, %xmm1, %xmm31142; AVX1-NEXT: vpsrad $31, %xmm3, %xmm11143; AVX1-NEXT: vpcmpeqd %xmm1, %xmm6, %xmm11144; AVX1-NEXT: vpackssdw %xmm5, %xmm1, %xmm11145; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm51146; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]1147; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm71148; AVX1-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[1,1,3,3]1149; AVX1-NEXT: vpmuldq %xmm6, %xmm8, %xmm61150; AVX1-NEXT: vpmuldq %xmm5, %xmm7, %xmm81151; AVX1-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]1152; AVX1-NEXT: vpblendw {{.*#+}} xmm6 = xmm8[0,1],xmm6[2,3],xmm8[4,5],xmm6[6,7]1153; AVX1-NEXT: vpmulld %xmm5, %xmm7, %xmm51154; AVX1-NEXT: vpsrad $31, %xmm5, %xmm71155; AVX1-NEXT: vpcmpeqd %xmm7, %xmm6, %xmm61156; AVX1-NEXT: vpshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]1157; AVX1-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[1,1,3,3]1158; AVX1-NEXT: vpmuldq %xmm7, %xmm8, %xmm71159; AVX1-NEXT: vpmuldq %xmm2, %xmm0, %xmm81160; AVX1-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]1161; AVX1-NEXT: vpblendw {{.*#+}} xmm7 = xmm8[0,1],xmm7[2,3],xmm8[4,5],xmm7[6,7]1162; AVX1-NEXT: vpmulld %xmm2, %xmm0, %xmm21163; AVX1-NEXT: vpsrad $31, %xmm2, %xmm01164; AVX1-NEXT: vpcmpeqd %xmm0, %xmm7, %xmm01165; AVX1-NEXT: vpackssdw %xmm6, %xmm0, %xmm01166; AVX1-NEXT: vpacksswb %xmm1, %xmm0, %xmm01167; AVX1-NEXT: vpcmpeqd %xmm6, %xmm6, %xmm61168; AVX1-NEXT: vpxor %xmm6, %xmm0, %xmm01169; AVX1-NEXT: vpmovsxbd %xmm0, %xmm71170; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]1171; AVX1-NEXT: vpmovsxbd %xmm0, %xmm01172; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm7, %ymm01173; AVX1-NEXT: vpacksswb %xmm1, %xmm1, %xmm11174; AVX1-NEXT: vpxor %xmm6, %xmm1, %xmm11175; AVX1-NEXT: vpmovsxbd %xmm1, %xmm61176; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]1177; AVX1-NEXT: vpmovsxbd %xmm1, %xmm11178; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm6, %ymm11179; AVX1-NEXT: vmovdqa %xmm4, 48(%rdi)1180; AVX1-NEXT: vmovdqa %xmm3, 32(%rdi)1181; AVX1-NEXT: vmovdqa %xmm5, 16(%rdi)1182; AVX1-NEXT: vmovdqa %xmm2, (%rdi)1183; AVX1-NEXT: retq1184;1185; AVX2-LABEL: smulo_v16i32:1186; AVX2: # %bb.0:1187; AVX2-NEXT: vpshufd {{.*#+}} ymm4 = ymm3[1,1,3,3,5,5,7,7]1188; AVX2-NEXT: vpshufd {{.*#+}} ymm5 = ymm1[1,1,3,3,5,5,7,7]1189; AVX2-NEXT: vpmuldq %ymm4, %ymm5, %ymm41190; AVX2-NEXT: vpmuldq %ymm3, %ymm1, %ymm51191; AVX2-NEXT: vpshufd {{.*#+}} ymm5 = ymm5[1,1,3,3,5,5,7,7]1192; AVX2-NEXT: vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7]1193; AVX2-NEXT: vpmulld %ymm3, %ymm1, %ymm31194; AVX2-NEXT: vpsrad $31, %ymm3, %ymm11195; AVX2-NEXT: vpcmpeqd %ymm1, %ymm4, %ymm11196; AVX2-NEXT: vpcmpeqd %ymm4, %ymm4, %ymm41197; AVX2-NEXT: vpxor %ymm4, %ymm1, %ymm11198; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm51199; AVX2-NEXT: vpackssdw %xmm5, %xmm1, %xmm11200; AVX2-NEXT: vpshufd {{.*#+}} ymm5 = ymm2[1,1,3,3,5,5,7,7]1201; AVX2-NEXT: vpshufd {{.*#+}} ymm6 = ymm0[1,1,3,3,5,5,7,7]1202; AVX2-NEXT: vpmuldq %ymm5, %ymm6, %ymm51203; AVX2-NEXT: vpmuldq %ymm2, %ymm0, %ymm61204; AVX2-NEXT: vpshufd {{.*#+}} ymm6 = ymm6[1,1,3,3,5,5,7,7]1205; AVX2-NEXT: vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7]1206; AVX2-NEXT: vpmulld %ymm2, %ymm0, %ymm21207; AVX2-NEXT: vpsrad $31, %ymm2, %ymm01208; AVX2-NEXT: vpcmpeqd %ymm0, %ymm5, %ymm01209; AVX2-NEXT: vpacksswb %xmm1, %xmm1, %xmm11210; AVX2-NEXT: vpmovsxbd %xmm1, %ymm11211; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4,8,8,8,8,12,12,12,12,16,16,16,16,20,20,20,20,24,24,24,24,28,28,28,28]1212; AVX2-NEXT: vpxor %ymm4, %ymm0, %ymm01213; AVX2-NEXT: vmovdqa %ymm3, 32(%rdi)1214; AVX2-NEXT: vmovdqa %ymm2, (%rdi)1215; AVX2-NEXT: retq1216;1217; AVX512-LABEL: smulo_v16i32:1218; AVX512: # %bb.0:1219; AVX512-NEXT: vpmuldq %zmm1, %zmm0, %zmm21220; AVX512-NEXT: vpshufd {{.*#+}} zmm3 = zmm1[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]1221; AVX512-NEXT: vpshufd {{.*#+}} zmm4 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]1222; AVX512-NEXT: vpmuldq %zmm3, %zmm4, %zmm31223; AVX512-NEXT: vpmovsxbd {{.*#+}} zmm4 = [1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31]1224; AVX512-NEXT: vpermi2d %zmm3, %zmm2, %zmm41225; AVX512-NEXT: vpmulld %zmm1, %zmm0, %zmm11226; AVX512-NEXT: vpsrad $31, %zmm1, %zmm01227; AVX512-NEXT: vpcmpneqd %zmm0, %zmm4, %k11228; AVX512-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -11229; AVX512-NEXT: vmovdqa64 %zmm1, (%rdi)1230; AVX512-NEXT: retq1231 %t = call {<16 x i32>, <16 x i1>} @llvm.smul.with.overflow.v16i32(<16 x i32> %a0, <16 x i32> %a1)1232 %val = extractvalue {<16 x i32>, <16 x i1>} %t, 01233 %obit = extractvalue {<16 x i32>, <16 x i1>} %t, 11234 %res = sext <16 x i1> %obit to <16 x i32>1235 store <16 x i32> %val, ptr %p21236 ret <16 x i32> %res1237}1238 1239define <16 x i32> @smulo_v16i8(<16 x i8> %a0, <16 x i8> %a1, ptr %p2) nounwind {1240; SSE2-LABEL: smulo_v16i8:1241; SSE2: # %bb.0:1242; SSE2-NEXT: pxor %xmm2, %xmm21243; SSE2-NEXT: pxor %xmm3, %xmm31244; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]1245; SSE2-NEXT: pxor %xmm5, %xmm51246; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]1247; SSE2-NEXT: pmulhw %xmm3, %xmm51248; SSE2-NEXT: movdqa %xmm5, %xmm31249; SSE2-NEXT: psrlw $8, %xmm31250; SSE2-NEXT: pxor %xmm6, %xmm61251; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3],xmm6[4],xmm1[4],xmm6[5],xmm1[5],xmm6[6],xmm1[6],xmm6[7],xmm1[7]1252; SSE2-NEXT: pxor %xmm4, %xmm41253; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]1254; SSE2-NEXT: pmulhw %xmm6, %xmm41255; SSE2-NEXT: movdqa %xmm4, %xmm01256; SSE2-NEXT: psrlw $8, %xmm01257; SSE2-NEXT: packuswb %xmm3, %xmm01258; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]1259; SSE2-NEXT: pand %xmm1, %xmm51260; SSE2-NEXT: pand %xmm1, %xmm41261; SSE2-NEXT: packuswb %xmm5, %xmm41262; SSE2-NEXT: pcmpgtb %xmm4, %xmm21263; SSE2-NEXT: pcmpeqb %xmm0, %xmm21264; SSE2-NEXT: pcmpeqd %xmm3, %xmm31265; SSE2-NEXT: pxor %xmm2, %xmm31266; SSE2-NEXT: movdqa %xmm3, %xmm01267; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]1268; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]1269; SSE2-NEXT: movdqa %xmm3, %xmm11270; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1271; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]1272; SSE2-NEXT: pslld $31, %xmm11273; SSE2-NEXT: psrad $31, %xmm11274; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1275; SSE2-NEXT: movdqa %xmm3, %xmm21276; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]1277; SSE2-NEXT: pslld $31, %xmm21278; SSE2-NEXT: psrad $31, %xmm21279; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]1280; SSE2-NEXT: pslld $31, %xmm31281; SSE2-NEXT: psrad $31, %xmm31282; SSE2-NEXT: movdqa %xmm4, (%rdi)1283; SSE2-NEXT: retq1284;1285; SSSE3-LABEL: smulo_v16i8:1286; SSSE3: # %bb.0:1287; SSSE3-NEXT: pxor %xmm2, %xmm21288; SSSE3-NEXT: pxor %xmm3, %xmm31289; SSSE3-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]1290; SSSE3-NEXT: pxor %xmm5, %xmm51291; SSSE3-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]1292; SSSE3-NEXT: pmulhw %xmm3, %xmm51293; SSSE3-NEXT: movdqa %xmm5, %xmm31294; SSSE3-NEXT: psrlw $8, %xmm31295; SSSE3-NEXT: pxor %xmm6, %xmm61296; SSSE3-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3],xmm6[4],xmm1[4],xmm6[5],xmm1[5],xmm6[6],xmm1[6],xmm6[7],xmm1[7]1297; SSSE3-NEXT: pxor %xmm4, %xmm41298; SSSE3-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]1299; SSSE3-NEXT: pmulhw %xmm6, %xmm41300; SSSE3-NEXT: movdqa %xmm4, %xmm01301; SSSE3-NEXT: psrlw $8, %xmm01302; SSSE3-NEXT: packuswb %xmm3, %xmm01303; SSSE3-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]1304; SSSE3-NEXT: pand %xmm1, %xmm51305; SSSE3-NEXT: pand %xmm1, %xmm41306; SSSE3-NEXT: packuswb %xmm5, %xmm41307; SSSE3-NEXT: pcmpgtb %xmm4, %xmm21308; SSSE3-NEXT: pcmpeqb %xmm0, %xmm21309; SSSE3-NEXT: pcmpeqd %xmm3, %xmm31310; SSSE3-NEXT: pxor %xmm2, %xmm31311; SSSE3-NEXT: movdqa %xmm3, %xmm01312; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]1313; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]1314; SSSE3-NEXT: movdqa %xmm3, %xmm11315; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1316; SSSE3-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]1317; SSSE3-NEXT: pslld $31, %xmm11318; SSSE3-NEXT: psrad $31, %xmm11319; SSSE3-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1320; SSSE3-NEXT: movdqa %xmm3, %xmm21321; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]1322; SSSE3-NEXT: pslld $31, %xmm21323; SSSE3-NEXT: psrad $31, %xmm21324; SSSE3-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]1325; SSSE3-NEXT: pslld $31, %xmm31326; SSSE3-NEXT: psrad $31, %xmm31327; SSSE3-NEXT: movdqa %xmm4, (%rdi)1328; SSSE3-NEXT: retq1329;1330; SSE41-LABEL: smulo_v16i8:1331; SSE41: # %bb.0:1332; SSE41-NEXT: pxor %xmm2, %xmm21333; SSE41-NEXT: pxor %xmm3, %xmm31334; SSE41-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]1335; SSE41-NEXT: pxor %xmm5, %xmm51336; SSE41-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]1337; SSE41-NEXT: pmulhw %xmm3, %xmm51338; SSE41-NEXT: movdqa %xmm5, %xmm31339; SSE41-NEXT: psrlw $8, %xmm31340; SSE41-NEXT: pxor %xmm6, %xmm61341; SSE41-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3],xmm6[4],xmm1[4],xmm6[5],xmm1[5],xmm6[6],xmm1[6],xmm6[7],xmm1[7]1342; SSE41-NEXT: pxor %xmm4, %xmm41343; SSE41-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]1344; SSE41-NEXT: pmulhw %xmm6, %xmm41345; SSE41-NEXT: movdqa %xmm4, %xmm01346; SSE41-NEXT: psrlw $8, %xmm01347; SSE41-NEXT: packuswb %xmm3, %xmm01348; SSE41-NEXT: pmovzxbw {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]1349; SSE41-NEXT: pand %xmm1, %xmm51350; SSE41-NEXT: pand %xmm1, %xmm41351; SSE41-NEXT: packuswb %xmm5, %xmm41352; SSE41-NEXT: pcmpgtb %xmm4, %xmm21353; SSE41-NEXT: pcmpeqb %xmm0, %xmm21354; SSE41-NEXT: pcmpeqd %xmm3, %xmm31355; SSE41-NEXT: pxor %xmm2, %xmm31356; SSE41-NEXT: pmovsxbd %xmm3, %xmm01357; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]1358; SSE41-NEXT: pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero1359; SSE41-NEXT: pslld $31, %xmm11360; SSE41-NEXT: psrad $31, %xmm11361; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]1362; SSE41-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero1363; SSE41-NEXT: pslld $31, %xmm21364; SSE41-NEXT: psrad $31, %xmm21365; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]1366; SSE41-NEXT: pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero1367; SSE41-NEXT: pslld $31, %xmm31368; SSE41-NEXT: psrad $31, %xmm31369; SSE41-NEXT: movdqa %xmm4, (%rdi)1370; SSE41-NEXT: retq1371;1372; AVX1-LABEL: smulo_v16i8:1373; AVX1: # %bb.0:1374; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm21375; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]1376; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]1377; AVX1-NEXT: vpmulhw %xmm3, %xmm4, %xmm31378; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm41379; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]1380; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]1381; AVX1-NEXT: vpmulhw %xmm1, %xmm0, %xmm01382; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm11383; AVX1-NEXT: vpackuswb %xmm4, %xmm1, %xmm11384; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]1385; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm31386; AVX1-NEXT: vpand %xmm4, %xmm0, %xmm01387; AVX1-NEXT: vpackuswb %xmm3, %xmm0, %xmm31388; AVX1-NEXT: vpcmpgtb %xmm3, %xmm2, %xmm01389; AVX1-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm01390; AVX1-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm11391; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm11392; AVX1-NEXT: vpmovsxbd %xmm1, %xmm01393; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]1394; AVX1-NEXT: vpmovsxbd %xmm2, %xmm21395; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm01396; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]1397; AVX1-NEXT: vpmovsxbd %xmm2, %xmm21398; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]1399; AVX1-NEXT: vpmovsxbd %xmm1, %xmm11400; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm11401; AVX1-NEXT: vmovdqa %xmm3, (%rdi)1402; AVX1-NEXT: retq1403;1404; AVX2-LABEL: smulo_v16i8:1405; AVX2: # %bb.0:1406; AVX2-NEXT: vpmovsxbw %xmm1, %ymm11407; AVX2-NEXT: vpmovsxbw %xmm0, %ymm01408; AVX2-NEXT: vpmullw %ymm1, %ymm0, %ymm01409; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm11410; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm21411; AVX2-NEXT: vpackuswb %xmm2, %xmm1, %xmm11412; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01413; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm21414; AVX2-NEXT: vpackuswb %xmm2, %xmm0, %xmm21415; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm01416; AVX2-NEXT: vpcmpgtb %xmm2, %xmm0, %xmm01417; AVX2-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm01418; AVX2-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm11419; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm11420; AVX2-NEXT: vpmovsxbd %xmm1, %ymm01421; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]1422; AVX2-NEXT: vpmovsxbd %xmm1, %ymm11423; AVX2-NEXT: vmovdqa %xmm2, (%rdi)1424; AVX2-NEXT: retq1425;1426; AVX512F-LABEL: smulo_v16i8:1427; AVX512F: # %bb.0:1428; AVX512F-NEXT: vpmovsxbw %xmm1, %ymm11429; AVX512F-NEXT: vpmovsxbw %xmm0, %ymm01430; AVX512F-NEXT: vpmullw %ymm1, %ymm0, %ymm11431; AVX512F-NEXT: vpsraw $8, %ymm1, %ymm01432; AVX512F-NEXT: vpmovsxwd %ymm0, %zmm01433; AVX512F-NEXT: vpsllw $8, %ymm1, %ymm21434; AVX512F-NEXT: vpsraw $15, %ymm2, %ymm21435; AVX512F-NEXT: vpmovsxwd %ymm2, %zmm21436; AVX512F-NEXT: vpcmpneqd %zmm0, %zmm2, %k11437; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -11438; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero1439; AVX512F-NEXT: vpmovdb %zmm1, (%rdi)1440; AVX512F-NEXT: retq1441;1442; AVX512BW-LABEL: smulo_v16i8:1443; AVX512BW: # %bb.0:1444; AVX512BW-NEXT: vpmovsxbw %xmm1, %ymm11445; AVX512BW-NEXT: vpmovsxbw %xmm0, %ymm01446; AVX512BW-NEXT: vpmullw %ymm1, %ymm0, %ymm11447; AVX512BW-NEXT: vpsraw $8, %ymm1, %ymm01448; AVX512BW-NEXT: vpsllw $8, %ymm1, %ymm21449; AVX512BW-NEXT: vpsraw $15, %ymm2, %ymm21450; AVX512BW-NEXT: vpcmpneqw %ymm0, %ymm2, %k11451; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -11452; AVX512BW-NEXT: vpmovwb %ymm1, (%rdi)1453; AVX512BW-NEXT: retq1454 %t = call {<16 x i8>, <16 x i1>} @llvm.smul.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)1455 %val = extractvalue {<16 x i8>, <16 x i1>} %t, 01456 %obit = extractvalue {<16 x i8>, <16 x i1>} %t, 11457 %res = sext <16 x i1> %obit to <16 x i32>1458 store <16 x i8> %val, ptr %p21459 ret <16 x i32> %res1460}1461 1462define <32 x i32> @smulo_v32i8(<32 x i8> %a0, <32 x i8> %a1, ptr %p2) nounwind {1463; SSE2-LABEL: smulo_v32i8:1464; SSE2: # %bb.0:1465; SSE2-NEXT: movq %rdi, %rax1466; SSE2-NEXT: pxor %xmm4, %xmm41467; SSE2-NEXT: pxor %xmm5, %xmm51468; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]1469; SSE2-NEXT: pxor %xmm6, %xmm61470; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm1[8],xmm6[9],xmm1[9],xmm6[10],xmm1[10],xmm6[11],xmm1[11],xmm6[12],xmm1[12],xmm6[13],xmm1[13],xmm6[14],xmm1[14],xmm6[15],xmm1[15]1471; SSE2-NEXT: pmulhw %xmm5, %xmm61472; SSE2-NEXT: movdqa %xmm6, %xmm51473; SSE2-NEXT: psrlw $8, %xmm51474; SSE2-NEXT: pxor %xmm7, %xmm71475; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]1476; SSE2-NEXT: pxor %xmm3, %xmm31477; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]1478; SSE2-NEXT: pmulhw %xmm7, %xmm31479; SSE2-NEXT: movdqa %xmm3, %xmm71480; SSE2-NEXT: psrlw $8, %xmm71481; SSE2-NEXT: packuswb %xmm5, %xmm71482; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]1483; SSE2-NEXT: pand %xmm5, %xmm61484; SSE2-NEXT: pand %xmm5, %xmm31485; SSE2-NEXT: packuswb %xmm6, %xmm31486; SSE2-NEXT: pxor %xmm1, %xmm11487; SSE2-NEXT: pcmpgtb %xmm3, %xmm11488; SSE2-NEXT: pcmpeqb %xmm7, %xmm11489; SSE2-NEXT: pcmpeqd %xmm6, %xmm61490; SSE2-NEXT: pxor %xmm6, %xmm11491; SSE2-NEXT: pxor %xmm7, %xmm71492; SSE2-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm2[8],xmm7[9],xmm2[9],xmm7[10],xmm2[10],xmm7[11],xmm2[11],xmm7[12],xmm2[12],xmm7[13],xmm2[13],xmm7[14],xmm2[14],xmm7[15],xmm2[15]1493; SSE2-NEXT: pxor %xmm8, %xmm81494; SSE2-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm0[8],xmm8[9],xmm0[9],xmm8[10],xmm0[10],xmm8[11],xmm0[11],xmm8[12],xmm0[12],xmm8[13],xmm0[13],xmm8[14],xmm0[14],xmm8[15],xmm0[15]1495; SSE2-NEXT: pmulhw %xmm7, %xmm81496; SSE2-NEXT: movdqa %xmm8, %xmm71497; SSE2-NEXT: psrlw $8, %xmm71498; SSE2-NEXT: pxor %xmm9, %xmm91499; SSE2-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3],xmm9[4],xmm2[4],xmm9[5],xmm2[5],xmm9[6],xmm2[6],xmm9[7],xmm2[7]1500; SSE2-NEXT: pxor %xmm2, %xmm21501; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]1502; SSE2-NEXT: pmulhw %xmm9, %xmm21503; SSE2-NEXT: movdqa %xmm2, %xmm01504; SSE2-NEXT: psrlw $8, %xmm01505; SSE2-NEXT: packuswb %xmm7, %xmm01506; SSE2-NEXT: pand %xmm5, %xmm81507; SSE2-NEXT: pand %xmm5, %xmm21508; SSE2-NEXT: packuswb %xmm8, %xmm21509; SSE2-NEXT: pcmpgtb %xmm2, %xmm41510; SSE2-NEXT: pcmpeqb %xmm0, %xmm41511; SSE2-NEXT: pxor %xmm6, %xmm41512; SSE2-NEXT: movdqa %xmm4, %xmm01513; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1514; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]1515; SSE2-NEXT: pslld $31, %xmm01516; SSE2-NEXT: psrad $31, %xmm01517; SSE2-NEXT: movdqa %xmm4, %xmm51518; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1519; SSE2-NEXT: movdqa %xmm5, %xmm61520; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3]1521; SSE2-NEXT: pslld $31, %xmm61522; SSE2-NEXT: psrad $31, %xmm61523; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]1524; SSE2-NEXT: pslld $31, %xmm51525; SSE2-NEXT: psrad $31, %xmm51526; SSE2-NEXT: movdqa %xmm1, %xmm71527; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1528; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4,4,5,5,6,6,7,7]1529; SSE2-NEXT: pslld $31, %xmm71530; SSE2-NEXT: psrad $31, %xmm71531; SSE2-NEXT: movdqa %xmm1, %xmm81532; SSE2-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1533; SSE2-NEXT: movdqa %xmm8, %xmm91534; SSE2-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3]1535; SSE2-NEXT: pslld $31, %xmm91536; SSE2-NEXT: psrad $31, %xmm91537; SSE2-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4,4,5,5,6,6,7,7]1538; SSE2-NEXT: pslld $31, %xmm81539; SSE2-NEXT: psrad $31, %xmm81540; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1541; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]1542; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1543; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]1544; SSE2-NEXT: movdqa %xmm3, 16(%rsi)1545; SSE2-NEXT: movdqa %xmm2, (%rsi)1546; SSE2-NEXT: movdqa %xmm1, 64(%rdi)1547; SSE2-NEXT: movdqa %xmm4, (%rdi)1548; SSE2-NEXT: movdqa %xmm8, 112(%rdi)1549; SSE2-NEXT: movdqa %xmm9, 96(%rdi)1550; SSE2-NEXT: movdqa %xmm7, 80(%rdi)1551; SSE2-NEXT: movdqa %xmm5, 48(%rdi)1552; SSE2-NEXT: movdqa %xmm6, 32(%rdi)1553; SSE2-NEXT: movdqa %xmm0, 16(%rdi)1554; SSE2-NEXT: retq1555;1556; SSSE3-LABEL: smulo_v32i8:1557; SSSE3: # %bb.0:1558; SSSE3-NEXT: movq %rdi, %rax1559; SSSE3-NEXT: pxor %xmm4, %xmm41560; SSSE3-NEXT: pxor %xmm5, %xmm51561; SSSE3-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]1562; SSSE3-NEXT: pxor %xmm6, %xmm61563; SSSE3-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm1[8],xmm6[9],xmm1[9],xmm6[10],xmm1[10],xmm6[11],xmm1[11],xmm6[12],xmm1[12],xmm6[13],xmm1[13],xmm6[14],xmm1[14],xmm6[15],xmm1[15]1564; SSSE3-NEXT: pmulhw %xmm5, %xmm61565; SSSE3-NEXT: movdqa %xmm6, %xmm51566; SSSE3-NEXT: psrlw $8, %xmm51567; SSSE3-NEXT: pxor %xmm7, %xmm71568; SSSE3-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]1569; SSSE3-NEXT: pxor %xmm3, %xmm31570; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]1571; SSSE3-NEXT: pmulhw %xmm7, %xmm31572; SSSE3-NEXT: movdqa %xmm3, %xmm71573; SSSE3-NEXT: psrlw $8, %xmm71574; SSSE3-NEXT: packuswb %xmm5, %xmm71575; SSSE3-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]1576; SSSE3-NEXT: pand %xmm5, %xmm61577; SSSE3-NEXT: pand %xmm5, %xmm31578; SSSE3-NEXT: packuswb %xmm6, %xmm31579; SSSE3-NEXT: pxor %xmm1, %xmm11580; SSSE3-NEXT: pcmpgtb %xmm3, %xmm11581; SSSE3-NEXT: pcmpeqb %xmm7, %xmm11582; SSSE3-NEXT: pcmpeqd %xmm6, %xmm61583; SSSE3-NEXT: pxor %xmm6, %xmm11584; SSSE3-NEXT: pxor %xmm7, %xmm71585; SSSE3-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm2[8],xmm7[9],xmm2[9],xmm7[10],xmm2[10],xmm7[11],xmm2[11],xmm7[12],xmm2[12],xmm7[13],xmm2[13],xmm7[14],xmm2[14],xmm7[15],xmm2[15]1586; SSSE3-NEXT: pxor %xmm8, %xmm81587; SSSE3-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm0[8],xmm8[9],xmm0[9],xmm8[10],xmm0[10],xmm8[11],xmm0[11],xmm8[12],xmm0[12],xmm8[13],xmm0[13],xmm8[14],xmm0[14],xmm8[15],xmm0[15]1588; SSSE3-NEXT: pmulhw %xmm7, %xmm81589; SSSE3-NEXT: movdqa %xmm8, %xmm71590; SSSE3-NEXT: psrlw $8, %xmm71591; SSSE3-NEXT: pxor %xmm9, %xmm91592; SSSE3-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3],xmm9[4],xmm2[4],xmm9[5],xmm2[5],xmm9[6],xmm2[6],xmm9[7],xmm2[7]1593; SSSE3-NEXT: pxor %xmm2, %xmm21594; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]1595; SSSE3-NEXT: pmulhw %xmm9, %xmm21596; SSSE3-NEXT: movdqa %xmm2, %xmm01597; SSSE3-NEXT: psrlw $8, %xmm01598; SSSE3-NEXT: packuswb %xmm7, %xmm01599; SSSE3-NEXT: pand %xmm5, %xmm81600; SSSE3-NEXT: pand %xmm5, %xmm21601; SSSE3-NEXT: packuswb %xmm8, %xmm21602; SSSE3-NEXT: pcmpgtb %xmm2, %xmm41603; SSSE3-NEXT: pcmpeqb %xmm0, %xmm41604; SSSE3-NEXT: pxor %xmm6, %xmm41605; SSSE3-NEXT: movdqa %xmm4, %xmm01606; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1607; SSSE3-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]1608; SSSE3-NEXT: pslld $31, %xmm01609; SSSE3-NEXT: psrad $31, %xmm01610; SSSE3-NEXT: movdqa %xmm4, %xmm51611; SSSE3-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1612; SSSE3-NEXT: movdqa %xmm5, %xmm61613; SSSE3-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3]1614; SSSE3-NEXT: pslld $31, %xmm61615; SSSE3-NEXT: psrad $31, %xmm61616; SSSE3-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]1617; SSSE3-NEXT: pslld $31, %xmm51618; SSSE3-NEXT: psrad $31, %xmm51619; SSSE3-NEXT: movdqa %xmm1, %xmm71620; SSSE3-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1621; SSSE3-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4,4,5,5,6,6,7,7]1622; SSSE3-NEXT: pslld $31, %xmm71623; SSSE3-NEXT: psrad $31, %xmm71624; SSSE3-NEXT: movdqa %xmm1, %xmm81625; SSSE3-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1626; SSSE3-NEXT: movdqa %xmm8, %xmm91627; SSSE3-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0,0,1,1,2,2,3,3]1628; SSSE3-NEXT: pslld $31, %xmm91629; SSSE3-NEXT: psrad $31, %xmm91630; SSSE3-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4,4,5,5,6,6,7,7]1631; SSSE3-NEXT: pslld $31, %xmm81632; SSSE3-NEXT: psrad $31, %xmm81633; SSSE3-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1634; SSSE3-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]1635; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1636; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]1637; SSSE3-NEXT: movdqa %xmm3, 16(%rsi)1638; SSSE3-NEXT: movdqa %xmm2, (%rsi)1639; SSSE3-NEXT: movdqa %xmm1, 64(%rdi)1640; SSSE3-NEXT: movdqa %xmm4, (%rdi)1641; SSSE3-NEXT: movdqa %xmm8, 112(%rdi)1642; SSSE3-NEXT: movdqa %xmm9, 96(%rdi)1643; SSSE3-NEXT: movdqa %xmm7, 80(%rdi)1644; SSSE3-NEXT: movdqa %xmm5, 48(%rdi)1645; SSSE3-NEXT: movdqa %xmm6, 32(%rdi)1646; SSSE3-NEXT: movdqa %xmm0, 16(%rdi)1647; SSSE3-NEXT: retq1648;1649; SSE41-LABEL: smulo_v32i8:1650; SSE41: # %bb.0:1651; SSE41-NEXT: movq %rdi, %rax1652; SSE41-NEXT: pxor %xmm4, %xmm41653; SSE41-NEXT: pxor %xmm5, %xmm51654; SSE41-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]1655; SSE41-NEXT: pxor %xmm6, %xmm61656; SSE41-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm1[8],xmm6[9],xmm1[9],xmm6[10],xmm1[10],xmm6[11],xmm1[11],xmm6[12],xmm1[12],xmm6[13],xmm1[13],xmm6[14],xmm1[14],xmm6[15],xmm1[15]1657; SSE41-NEXT: pmulhw %xmm5, %xmm61658; SSE41-NEXT: movdqa %xmm6, %xmm51659; SSE41-NEXT: psrlw $8, %xmm51660; SSE41-NEXT: pxor %xmm7, %xmm71661; SSE41-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]1662; SSE41-NEXT: pxor %xmm3, %xmm31663; SSE41-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]1664; SSE41-NEXT: pmulhw %xmm7, %xmm31665; SSE41-NEXT: movdqa %xmm3, %xmm71666; SSE41-NEXT: psrlw $8, %xmm71667; SSE41-NEXT: packuswb %xmm5, %xmm71668; SSE41-NEXT: pmovzxbw {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255]1669; SSE41-NEXT: pand %xmm5, %xmm61670; SSE41-NEXT: pand %xmm5, %xmm31671; SSE41-NEXT: packuswb %xmm6, %xmm31672; SSE41-NEXT: pxor %xmm1, %xmm11673; SSE41-NEXT: pcmpgtb %xmm3, %xmm11674; SSE41-NEXT: pcmpeqb %xmm7, %xmm11675; SSE41-NEXT: pcmpeqd %xmm6, %xmm61676; SSE41-NEXT: pxor %xmm6, %xmm11677; SSE41-NEXT: pxor %xmm7, %xmm71678; SSE41-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm2[8],xmm7[9],xmm2[9],xmm7[10],xmm2[10],xmm7[11],xmm2[11],xmm7[12],xmm2[12],xmm7[13],xmm2[13],xmm7[14],xmm2[14],xmm7[15],xmm2[15]1679; SSE41-NEXT: pxor %xmm8, %xmm81680; SSE41-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm0[8],xmm8[9],xmm0[9],xmm8[10],xmm0[10],xmm8[11],xmm0[11],xmm8[12],xmm0[12],xmm8[13],xmm0[13],xmm8[14],xmm0[14],xmm8[15],xmm0[15]1681; SSE41-NEXT: pmulhw %xmm7, %xmm81682; SSE41-NEXT: movdqa %xmm8, %xmm71683; SSE41-NEXT: psrlw $8, %xmm71684; SSE41-NEXT: pxor %xmm9, %xmm91685; SSE41-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3],xmm9[4],xmm2[4],xmm9[5],xmm2[5],xmm9[6],xmm2[6],xmm9[7],xmm2[7]1686; SSE41-NEXT: pxor %xmm2, %xmm21687; SSE41-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]1688; SSE41-NEXT: pmulhw %xmm9, %xmm21689; SSE41-NEXT: movdqa %xmm2, %xmm01690; SSE41-NEXT: psrlw $8, %xmm01691; SSE41-NEXT: packuswb %xmm7, %xmm01692; SSE41-NEXT: pand %xmm5, %xmm81693; SSE41-NEXT: pand %xmm5, %xmm21694; SSE41-NEXT: packuswb %xmm8, %xmm21695; SSE41-NEXT: pcmpgtb %xmm2, %xmm41696; SSE41-NEXT: pcmpeqb %xmm0, %xmm41697; SSE41-NEXT: pxor %xmm6, %xmm41698; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,1,1]1699; SSE41-NEXT: pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero1700; SSE41-NEXT: pslld $31, %xmm01701; SSE41-NEXT: psrad $31, %xmm01702; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]1703; SSE41-NEXT: pmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero1704; SSE41-NEXT: pslld $31, %xmm51705; SSE41-NEXT: psrad $31, %xmm51706; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]1707; SSE41-NEXT: pmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero1708; SSE41-NEXT: pslld $31, %xmm61709; SSE41-NEXT: psrad $31, %xmm61710; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm1[1,1,1,1]1711; SSE41-NEXT: pmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero1712; SSE41-NEXT: pslld $31, %xmm71713; SSE41-NEXT: psrad $31, %xmm71714; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm1[2,3,2,3]1715; SSE41-NEXT: pmovzxbd {{.*#+}} xmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero1716; SSE41-NEXT: pslld $31, %xmm81717; SSE41-NEXT: psrad $31, %xmm81718; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm1[3,3,3,3]1719; SSE41-NEXT: pmovzxbd {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero1720; SSE41-NEXT: pslld $31, %xmm91721; SSE41-NEXT: psrad $31, %xmm91722; SSE41-NEXT: pmovsxbd %xmm4, %xmm41723; SSE41-NEXT: pmovsxbd %xmm1, %xmm11724; SSE41-NEXT: movdqa %xmm3, 16(%rsi)1725; SSE41-NEXT: movdqa %xmm2, (%rsi)1726; SSE41-NEXT: movdqa %xmm1, 64(%rdi)1727; SSE41-NEXT: movdqa %xmm4, (%rdi)1728; SSE41-NEXT: movdqa %xmm9, 112(%rdi)1729; SSE41-NEXT: movdqa %xmm8, 96(%rdi)1730; SSE41-NEXT: movdqa %xmm7, 80(%rdi)1731; SSE41-NEXT: movdqa %xmm6, 48(%rdi)1732; SSE41-NEXT: movdqa %xmm5, 32(%rdi)1733; SSE41-NEXT: movdqa %xmm0, 16(%rdi)1734; SSE41-NEXT: retq1735;1736; AVX1-LABEL: smulo_v32i8:1737; AVX1: # %bb.0:1738; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm31739; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm21740; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]1741; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm51742; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15]1743; AVX1-NEXT: vpmulhw %xmm4, %xmm6, %xmm41744; AVX1-NEXT: vpsrlw $8, %xmm4, %xmm61745; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]1746; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm5 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7]1747; AVX1-NEXT: vpmulhw %xmm3, %xmm5, %xmm31748; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm51749; AVX1-NEXT: vpackuswb %xmm6, %xmm5, %xmm51750; AVX1-NEXT: vbroadcastss {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]1751; AVX1-NEXT: vpand %xmm6, %xmm4, %xmm41752; AVX1-NEXT: vpand %xmm6, %xmm3, %xmm31753; AVX1-NEXT: vpackuswb %xmm4, %xmm3, %xmm41754; AVX1-NEXT: vpcmpgtb %xmm4, %xmm2, %xmm31755; AVX1-NEXT: vpcmpeqb %xmm5, %xmm3, %xmm31756; AVX1-NEXT: vpcmpeqd %xmm5, %xmm5, %xmm51757; AVX1-NEXT: vpxor %xmm5, %xmm3, %xmm31758; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]1759; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm8 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]1760; AVX1-NEXT: vpmulhw %xmm7, %xmm8, %xmm71761; AVX1-NEXT: vpsrlw $8, %xmm7, %xmm81762; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]1763; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]1764; AVX1-NEXT: vpmulhw %xmm1, %xmm0, %xmm01765; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm11766; AVX1-NEXT: vpackuswb %xmm8, %xmm1, %xmm11767; AVX1-NEXT: vpand %xmm6, %xmm7, %xmm71768; AVX1-NEXT: vpand %xmm6, %xmm0, %xmm01769; AVX1-NEXT: vpackuswb %xmm7, %xmm0, %xmm61770; AVX1-NEXT: vpcmpgtb %xmm6, %xmm2, %xmm01771; AVX1-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm01772; AVX1-NEXT: vpxor %xmm5, %xmm0, %xmm11773; AVX1-NEXT: vpmovsxbd %xmm1, %xmm01774; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]1775; AVX1-NEXT: vpmovsxbd %xmm2, %xmm21776; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm01777; AVX1-NEXT: vpmovsxbd %xmm3, %xmm21778; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]1779; AVX1-NEXT: vpmovsxbd %xmm5, %xmm51780; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm2, %ymm21781; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]1782; AVX1-NEXT: vpmovsxbd %xmm5, %xmm51783; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]1784; AVX1-NEXT: vpmovsxbd %xmm1, %xmm11785; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm5, %ymm11786; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]1787; AVX1-NEXT: vpmovsxbd %xmm5, %xmm51788; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]1789; AVX1-NEXT: vpmovsxbd %xmm3, %xmm31790; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm5, %ymm31791; AVX1-NEXT: vmovdqa %xmm4, 16(%rdi)1792; AVX1-NEXT: vmovdqa %xmm6, (%rdi)1793; AVX1-NEXT: retq1794;1795; AVX2-LABEL: smulo_v32i8:1796; AVX2: # %bb.0:1797; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm21798; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15],ymm2[24],ymm1[24],ymm2[25],ymm1[25],ymm2[26],ymm1[26],ymm2[27],ymm1[27],ymm2[28],ymm1[28],ymm2[29],ymm1[29],ymm2[30],ymm1[30],ymm2[31],ymm1[31]1799; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]1800; AVX2-NEXT: vpmulhw %ymm3, %ymm4, %ymm31801; AVX2-NEXT: vpsrlw $8, %ymm3, %ymm41802; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]1803; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]1804; AVX2-NEXT: vpmulhw %ymm1, %ymm0, %ymm01805; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm11806; AVX2-NEXT: vpackuswb %ymm4, %ymm1, %ymm11807; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]1808; AVX2-NEXT: vpand %ymm4, %ymm3, %ymm31809; AVX2-NEXT: vpand %ymm4, %ymm0, %ymm01810; AVX2-NEXT: vpackuswb %ymm3, %ymm0, %ymm41811; AVX2-NEXT: vpcmpgtb %ymm4, %ymm2, %ymm01812; AVX2-NEXT: vpcmpeqb %ymm1, %ymm0, %ymm01813; AVX2-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm11814; AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm11815; AVX2-NEXT: vpmovsxbd %xmm1, %ymm01816; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm31817; AVX2-NEXT: vpmovsxbd %xmm3, %ymm21818; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]1819; AVX2-NEXT: vpmovsxbd %xmm1, %ymm11820; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]1821; AVX2-NEXT: vpmovsxbd %xmm3, %ymm31822; AVX2-NEXT: vmovdqa %ymm4, (%rdi)1823; AVX2-NEXT: retq1824;1825; AVX512F-LABEL: smulo_v32i8:1826; AVX512F: # %bb.0:1827; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm21828; AVX512F-NEXT: vpmovsxbw %xmm2, %ymm21829; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm31830; AVX512F-NEXT: vpmovsxbw %xmm3, %ymm31831; AVX512F-NEXT: vpmullw %ymm2, %ymm3, %ymm21832; AVX512F-NEXT: vpsraw $8, %ymm2, %ymm31833; AVX512F-NEXT: vpmovsxwd %ymm3, %zmm31834; AVX512F-NEXT: vpsllw $8, %ymm2, %ymm41835; AVX512F-NEXT: vpsraw $15, %ymm4, %ymm41836; AVX512F-NEXT: vpmovsxwd %ymm4, %zmm41837; AVX512F-NEXT: vpcmpneqd %zmm3, %zmm4, %k11838; AVX512F-NEXT: vpmovsxbw %xmm1, %ymm11839; AVX512F-NEXT: vpmovsxbw %xmm0, %ymm01840; AVX512F-NEXT: vpmullw %ymm1, %ymm0, %ymm31841; AVX512F-NEXT: vpsraw $8, %ymm3, %ymm01842; AVX512F-NEXT: vpmovsxwd %ymm0, %zmm01843; AVX512F-NEXT: vpsllw $8, %ymm3, %ymm11844; AVX512F-NEXT: vpsraw $15, %ymm1, %ymm11845; AVX512F-NEXT: vpmovsxwd %ymm1, %zmm11846; AVX512F-NEXT: vpcmpneqd %zmm0, %zmm1, %k21847; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 {%k2} {z} = -11848; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k1} {z} = -11849; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero1850; AVX512F-NEXT: vpmovdb %zmm2, 16(%rdi)1851; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero1852; AVX512F-NEXT: vpmovdb %zmm2, (%rdi)1853; AVX512F-NEXT: retq1854;1855; AVX512BW-LABEL: smulo_v32i8:1856; AVX512BW: # %bb.0:1857; AVX512BW-NEXT: vpmovsxbw %ymm1, %zmm11858; AVX512BW-NEXT: vpmovsxbw %ymm0, %zmm01859; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm21860; AVX512BW-NEXT: vpsraw $8, %zmm2, %zmm01861; AVX512BW-NEXT: vpsllw $8, %zmm2, %zmm11862; AVX512BW-NEXT: vpsraw $15, %zmm1, %zmm11863; AVX512BW-NEXT: vpcmpneqw %zmm0, %zmm1, %k11864; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -11865; AVX512BW-NEXT: kshiftrd $16, %k1, %k11866; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 {%k1} {z} = -11867; AVX512BW-NEXT: vpmovwb %zmm2, (%rdi)1868; AVX512BW-NEXT: retq1869 %t = call {<32 x i8>, <32 x i1>} @llvm.smul.with.overflow.v32i8(<32 x i8> %a0, <32 x i8> %a1)1870 %val = extractvalue {<32 x i8>, <32 x i1>} %t, 01871 %obit = extractvalue {<32 x i8>, <32 x i1>} %t, 11872 %res = sext <32 x i1> %obit to <32 x i32>1873 store <32 x i8> %val, ptr %p21874 ret <32 x i32> %res1875}1876 1877define <64 x i32> @smulo_v64i8(<64 x i8> %a0, <64 x i8> %a1, ptr %p2) nounwind {1878; SSE2-LABEL: smulo_v64i8:1879; SSE2: # %bb.0:1880; SSE2-NEXT: movq %rdi, %rax1881; SSE2-NEXT: pxor %xmm8, %xmm81882; SSE2-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15]1883; SSE2-NEXT: pxor %xmm10, %xmm101884; SSE2-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm3[8],xmm10[9],xmm3[9],xmm10[10],xmm3[10],xmm10[11],xmm3[11],xmm10[12],xmm3[12],xmm10[13],xmm3[13],xmm10[14],xmm3[14],xmm10[15],xmm3[15]1885; SSE2-NEXT: pmulhw %xmm8, %xmm101886; SSE2-NEXT: movdqa %xmm10, %xmm81887; SSE2-NEXT: psrlw $8, %xmm81888; SSE2-NEXT: pxor %xmm9, %xmm91889; SSE2-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3],xmm9[4],xmm7[4],xmm9[5],xmm7[5],xmm9[6],xmm7[6],xmm9[7],xmm7[7]1890; SSE2-NEXT: pxor %xmm7, %xmm71891; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]1892; SSE2-NEXT: pmulhw %xmm9, %xmm71893; SSE2-NEXT: movdqa %xmm7, %xmm111894; SSE2-NEXT: psrlw $8, %xmm111895; SSE2-NEXT: packuswb %xmm8, %xmm111896; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]1897; SSE2-NEXT: pand %xmm9, %xmm101898; SSE2-NEXT: pand %xmm9, %xmm71899; SSE2-NEXT: packuswb %xmm10, %xmm71900; SSE2-NEXT: pxor %xmm3, %xmm31901; SSE2-NEXT: pcmpgtb %xmm7, %xmm31902; SSE2-NEXT: pcmpeqb %xmm11, %xmm31903; SSE2-NEXT: pxor %xmm8, %xmm81904; SSE2-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm6[8],xmm8[9],xmm6[9],xmm8[10],xmm6[10],xmm8[11],xmm6[11],xmm8[12],xmm6[12],xmm8[13],xmm6[13],xmm8[14],xmm6[14],xmm8[15],xmm6[15]1905; SSE2-NEXT: pxor %xmm10, %xmm101906; SSE2-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm2[8],xmm10[9],xmm2[9],xmm10[10],xmm2[10],xmm10[11],xmm2[11],xmm10[12],xmm2[12],xmm10[13],xmm2[13],xmm10[14],xmm2[14],xmm10[15],xmm2[15]1907; SSE2-NEXT: pmulhw %xmm8, %xmm101908; SSE2-NEXT: pxor %xmm8, %xmm81909; SSE2-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3],xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]1910; SSE2-NEXT: pxor %xmm6, %xmm61911; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]1912; SSE2-NEXT: movdqa %xmm10, %xmm21913; SSE2-NEXT: psrlw $8, %xmm21914; SSE2-NEXT: pmulhw %xmm8, %xmm61915; SSE2-NEXT: movdqa %xmm6, %xmm81916; SSE2-NEXT: psrlw $8, %xmm81917; SSE2-NEXT: packuswb %xmm2, %xmm81918; SSE2-NEXT: pand %xmm9, %xmm101919; SSE2-NEXT: pand %xmm9, %xmm61920; SSE2-NEXT: packuswb %xmm10, %xmm61921; SSE2-NEXT: pxor %xmm2, %xmm21922; SSE2-NEXT: pcmpgtb %xmm6, %xmm21923; SSE2-NEXT: pcmpeqb %xmm8, %xmm21924; SSE2-NEXT: pxor %xmm8, %xmm81925; SSE2-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm5[8],xmm8[9],xmm5[9],xmm8[10],xmm5[10],xmm8[11],xmm5[11],xmm8[12],xmm5[12],xmm8[13],xmm5[13],xmm8[14],xmm5[14],xmm8[15],xmm5[15]1926; SSE2-NEXT: pxor %xmm10, %xmm101927; SSE2-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm1[8],xmm10[9],xmm1[9],xmm10[10],xmm1[10],xmm10[11],xmm1[11],xmm10[12],xmm1[12],xmm10[13],xmm1[13],xmm10[14],xmm1[14],xmm10[15],xmm1[15]1928; SSE2-NEXT: pmulhw %xmm8, %xmm101929; SSE2-NEXT: pxor %xmm8, %xmm81930; SSE2-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3],xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7]1931; SSE2-NEXT: pxor %xmm5, %xmm51932; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]1933; SSE2-NEXT: movdqa %xmm10, %xmm11934; SSE2-NEXT: psrlw $8, %xmm11935; SSE2-NEXT: pmulhw %xmm8, %xmm51936; SSE2-NEXT: movdqa %xmm5, %xmm81937; SSE2-NEXT: psrlw $8, %xmm81938; SSE2-NEXT: packuswb %xmm1, %xmm81939; SSE2-NEXT: pand %xmm9, %xmm101940; SSE2-NEXT: pand %xmm9, %xmm51941; SSE2-NEXT: packuswb %xmm10, %xmm51942; SSE2-NEXT: pxor %xmm1, %xmm11943; SSE2-NEXT: pcmpgtb %xmm5, %xmm11944; SSE2-NEXT: pcmpeqb %xmm8, %xmm11945; SSE2-NEXT: pxor %xmm8, %xmm81946; SSE2-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm4[8],xmm8[9],xmm4[9],xmm8[10],xmm4[10],xmm8[11],xmm4[11],xmm8[12],xmm4[12],xmm8[13],xmm4[13],xmm8[14],xmm4[14],xmm8[15],xmm4[15]1947; SSE2-NEXT: pxor %xmm10, %xmm101948; SSE2-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm0[8],xmm10[9],xmm0[9],xmm10[10],xmm0[10],xmm10[11],xmm0[11],xmm10[12],xmm0[12],xmm10[13],xmm0[13],xmm10[14],xmm0[14],xmm10[15],xmm0[15]1949; SSE2-NEXT: pmulhw %xmm8, %xmm101950; SSE2-NEXT: pxor %xmm11, %xmm111951; SSE2-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3],xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7]1952; SSE2-NEXT: pxor %xmm8, %xmm81953; SSE2-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3],xmm8[4],xmm0[4],xmm8[5],xmm0[5],xmm8[6],xmm0[6],xmm8[7],xmm0[7]1954; SSE2-NEXT: movdqa %xmm10, %xmm01955; SSE2-NEXT: psrlw $8, %xmm01956; SSE2-NEXT: pmulhw %xmm11, %xmm81957; SSE2-NEXT: movdqa %xmm8, %xmm111958; SSE2-NEXT: psrlw $8, %xmm111959; SSE2-NEXT: packuswb %xmm0, %xmm111960; SSE2-NEXT: pand %xmm9, %xmm101961; SSE2-NEXT: pand %xmm9, %xmm81962; SSE2-NEXT: packuswb %xmm10, %xmm81963; SSE2-NEXT: pxor %xmm4, %xmm41964; SSE2-NEXT: pcmpgtb %xmm8, %xmm41965; SSE2-NEXT: pcmpeqb %xmm11, %xmm41966; SSE2-NEXT: pcmpeqd %xmm0, %xmm01967; SSE2-NEXT: pxor %xmm0, %xmm31968; SSE2-NEXT: pxor %xmm0, %xmm21969; SSE2-NEXT: pxor %xmm0, %xmm11970; SSE2-NEXT: pxor %xmm0, %xmm41971; SSE2-NEXT: movdqa %xmm4, %xmm01972; SSE2-NEXT: movdqa %xmm7, 48(%rsi)1973; SSE2-NEXT: movdqa %xmm1, %xmm71974; SSE2-NEXT: movdqa %xmm6, 32(%rsi)1975; SSE2-NEXT: movdqa %xmm2, %xmm61976; SSE2-NEXT: movdqa %xmm5, 16(%rsi)1977; SSE2-NEXT: movdqa %xmm3, %xmm51978; SSE2-NEXT: movdqa %xmm8, (%rsi)1979; SSE2-NEXT: movdqa %xmm3, %xmm81980; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1981; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]1982; SSE2-NEXT: movdqa %xmm3, 192(%rdi)1983; SSE2-NEXT: movdqa %xmm2, %xmm31984; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1985; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]1986; SSE2-NEXT: movdqa %xmm2, 128(%rdi)1987; SSE2-NEXT: movdqa %xmm1, %xmm21988; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1989; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]1990; SSE2-NEXT: movdqa %xmm1, 64(%rdi)1991; SSE2-NEXT: movdqa %xmm4, %xmm11992; SSE2-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1993; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1994; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]1995; SSE2-NEXT: movdqa %xmm4, (%rdi)1996; SSE2-NEXT: movdqa %xmm8, %xmm41997; SSE2-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3]1998; SSE2-NEXT: pslld $31, %xmm81999; SSE2-NEXT: psrad $31, %xmm82000; SSE2-NEXT: movdqa %xmm8, 224(%rdi)2001; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]2002; SSE2-NEXT: pslld $31, %xmm42003; SSE2-NEXT: psrad $31, %xmm42004; SSE2-NEXT: movdqa %xmm4, 240(%rdi)2005; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2006; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]2007; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]2008; SSE2-NEXT: pslld $31, %xmm52009; SSE2-NEXT: psrad $31, %xmm52010; SSE2-NEXT: movdqa %xmm5, 208(%rdi)2011; SSE2-NEXT: movdqa %xmm3, %xmm42012; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]2013; SSE2-NEXT: pslld $31, %xmm32014; SSE2-NEXT: psrad $31, %xmm32015; SSE2-NEXT: movdqa %xmm3, 160(%rdi)2016; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]2017; SSE2-NEXT: pslld $31, %xmm42018; SSE2-NEXT: psrad $31, %xmm42019; SSE2-NEXT: movdqa %xmm4, 176(%rdi)2020; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2021; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]2022; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4,4,5,5,6,6,7,7]2023; SSE2-NEXT: pslld $31, %xmm62024; SSE2-NEXT: psrad $31, %xmm62025; SSE2-NEXT: movdqa %xmm6, 144(%rdi)2026; SSE2-NEXT: movdqa %xmm2, %xmm32027; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]2028; SSE2-NEXT: pslld $31, %xmm22029; SSE2-NEXT: psrad $31, %xmm22030; SSE2-NEXT: movdqa %xmm2, 96(%rdi)2031; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]2032; SSE2-NEXT: pslld $31, %xmm32033; SSE2-NEXT: psrad $31, %xmm32034; SSE2-NEXT: movdqa %xmm3, 112(%rdi)2035; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2036; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]2037; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4,4,5,5,6,6,7,7]2038; SSE2-NEXT: pslld $31, %xmm72039; SSE2-NEXT: psrad $31, %xmm72040; SSE2-NEXT: movdqa %xmm7, 80(%rdi)2041; SSE2-NEXT: movdqa %xmm1, %xmm22042; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]2043; SSE2-NEXT: pslld $31, %xmm12044; SSE2-NEXT: psrad $31, %xmm12045; SSE2-NEXT: movdqa %xmm1, 32(%rdi)2046; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]2047; SSE2-NEXT: pslld $31, %xmm22048; SSE2-NEXT: psrad $31, %xmm22049; SSE2-NEXT: movdqa %xmm2, 48(%rdi)2050; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]2051; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]2052; SSE2-NEXT: pslld $31, %xmm02053; SSE2-NEXT: psrad $31, %xmm02054; SSE2-NEXT: movdqa %xmm0, 16(%rdi)2055; SSE2-NEXT: retq2056;2057; SSSE3-LABEL: smulo_v64i8:2058; SSSE3: # %bb.0:2059; SSSE3-NEXT: movq %rdi, %rax2060; SSSE3-NEXT: pxor %xmm8, %xmm82061; SSSE3-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15]2062; SSSE3-NEXT: pxor %xmm10, %xmm102063; SSSE3-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm3[8],xmm10[9],xmm3[9],xmm10[10],xmm3[10],xmm10[11],xmm3[11],xmm10[12],xmm3[12],xmm10[13],xmm3[13],xmm10[14],xmm3[14],xmm10[15],xmm3[15]2064; SSSE3-NEXT: pmulhw %xmm8, %xmm102065; SSSE3-NEXT: movdqa %xmm10, %xmm82066; SSSE3-NEXT: psrlw $8, %xmm82067; SSSE3-NEXT: pxor %xmm9, %xmm92068; SSSE3-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3],xmm9[4],xmm7[4],xmm9[5],xmm7[5],xmm9[6],xmm7[6],xmm9[7],xmm7[7]2069; SSSE3-NEXT: pxor %xmm7, %xmm72070; SSSE3-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]2071; SSSE3-NEXT: pmulhw %xmm9, %xmm72072; SSSE3-NEXT: movdqa %xmm7, %xmm112073; SSSE3-NEXT: psrlw $8, %xmm112074; SSSE3-NEXT: packuswb %xmm8, %xmm112075; SSSE3-NEXT: movdqa {{.*#+}} xmm9 = [255,255,255,255,255,255,255,255]2076; SSSE3-NEXT: pand %xmm9, %xmm102077; SSSE3-NEXT: pand %xmm9, %xmm72078; SSSE3-NEXT: packuswb %xmm10, %xmm72079; SSSE3-NEXT: pxor %xmm3, %xmm32080; SSSE3-NEXT: pcmpgtb %xmm7, %xmm32081; SSSE3-NEXT: pcmpeqb %xmm11, %xmm32082; SSSE3-NEXT: pxor %xmm8, %xmm82083; SSSE3-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm6[8],xmm8[9],xmm6[9],xmm8[10],xmm6[10],xmm8[11],xmm6[11],xmm8[12],xmm6[12],xmm8[13],xmm6[13],xmm8[14],xmm6[14],xmm8[15],xmm6[15]2084; SSSE3-NEXT: pxor %xmm10, %xmm102085; SSSE3-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm2[8],xmm10[9],xmm2[9],xmm10[10],xmm2[10],xmm10[11],xmm2[11],xmm10[12],xmm2[12],xmm10[13],xmm2[13],xmm10[14],xmm2[14],xmm10[15],xmm2[15]2086; SSSE3-NEXT: pmulhw %xmm8, %xmm102087; SSSE3-NEXT: pxor %xmm8, %xmm82088; SSSE3-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3],xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]2089; SSSE3-NEXT: pxor %xmm6, %xmm62090; SSSE3-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]2091; SSSE3-NEXT: movdqa %xmm10, %xmm22092; SSSE3-NEXT: psrlw $8, %xmm22093; SSSE3-NEXT: pmulhw %xmm8, %xmm62094; SSSE3-NEXT: movdqa %xmm6, %xmm82095; SSSE3-NEXT: psrlw $8, %xmm82096; SSSE3-NEXT: packuswb %xmm2, %xmm82097; SSSE3-NEXT: pand %xmm9, %xmm102098; SSSE3-NEXT: pand %xmm9, %xmm62099; SSSE3-NEXT: packuswb %xmm10, %xmm62100; SSSE3-NEXT: pxor %xmm2, %xmm22101; SSSE3-NEXT: pcmpgtb %xmm6, %xmm22102; SSSE3-NEXT: pcmpeqb %xmm8, %xmm22103; SSSE3-NEXT: pxor %xmm8, %xmm82104; SSSE3-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm5[8],xmm8[9],xmm5[9],xmm8[10],xmm5[10],xmm8[11],xmm5[11],xmm8[12],xmm5[12],xmm8[13],xmm5[13],xmm8[14],xmm5[14],xmm8[15],xmm5[15]2105; SSSE3-NEXT: pxor %xmm10, %xmm102106; SSSE3-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm1[8],xmm10[9],xmm1[9],xmm10[10],xmm1[10],xmm10[11],xmm1[11],xmm10[12],xmm1[12],xmm10[13],xmm1[13],xmm10[14],xmm1[14],xmm10[15],xmm1[15]2107; SSSE3-NEXT: pmulhw %xmm8, %xmm102108; SSSE3-NEXT: pxor %xmm8, %xmm82109; SSSE3-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3],xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7]2110; SSSE3-NEXT: pxor %xmm5, %xmm52111; SSSE3-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]2112; SSSE3-NEXT: movdqa %xmm10, %xmm12113; SSSE3-NEXT: psrlw $8, %xmm12114; SSSE3-NEXT: pmulhw %xmm8, %xmm52115; SSSE3-NEXT: movdqa %xmm5, %xmm82116; SSSE3-NEXT: psrlw $8, %xmm82117; SSSE3-NEXT: packuswb %xmm1, %xmm82118; SSSE3-NEXT: pand %xmm9, %xmm102119; SSSE3-NEXT: pand %xmm9, %xmm52120; SSSE3-NEXT: packuswb %xmm10, %xmm52121; SSSE3-NEXT: pxor %xmm1, %xmm12122; SSSE3-NEXT: pcmpgtb %xmm5, %xmm12123; SSSE3-NEXT: pcmpeqb %xmm8, %xmm12124; SSSE3-NEXT: pxor %xmm8, %xmm82125; SSSE3-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm4[8],xmm8[9],xmm4[9],xmm8[10],xmm4[10],xmm8[11],xmm4[11],xmm8[12],xmm4[12],xmm8[13],xmm4[13],xmm8[14],xmm4[14],xmm8[15],xmm4[15]2126; SSSE3-NEXT: pxor %xmm10, %xmm102127; SSSE3-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm0[8],xmm10[9],xmm0[9],xmm10[10],xmm0[10],xmm10[11],xmm0[11],xmm10[12],xmm0[12],xmm10[13],xmm0[13],xmm10[14],xmm0[14],xmm10[15],xmm0[15]2128; SSSE3-NEXT: pmulhw %xmm8, %xmm102129; SSSE3-NEXT: pxor %xmm11, %xmm112130; SSSE3-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3],xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7]2131; SSSE3-NEXT: pxor %xmm8, %xmm82132; SSSE3-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3],xmm8[4],xmm0[4],xmm8[5],xmm0[5],xmm8[6],xmm0[6],xmm8[7],xmm0[7]2133; SSSE3-NEXT: movdqa %xmm10, %xmm02134; SSSE3-NEXT: psrlw $8, %xmm02135; SSSE3-NEXT: pmulhw %xmm11, %xmm82136; SSSE3-NEXT: movdqa %xmm8, %xmm112137; SSSE3-NEXT: psrlw $8, %xmm112138; SSSE3-NEXT: packuswb %xmm0, %xmm112139; SSSE3-NEXT: pand %xmm9, %xmm102140; SSSE3-NEXT: pand %xmm9, %xmm82141; SSSE3-NEXT: packuswb %xmm10, %xmm82142; SSSE3-NEXT: pxor %xmm4, %xmm42143; SSSE3-NEXT: pcmpgtb %xmm8, %xmm42144; SSSE3-NEXT: pcmpeqb %xmm11, %xmm42145; SSSE3-NEXT: pcmpeqd %xmm0, %xmm02146; SSSE3-NEXT: pxor %xmm0, %xmm32147; SSSE3-NEXT: pxor %xmm0, %xmm22148; SSSE3-NEXT: pxor %xmm0, %xmm12149; SSSE3-NEXT: pxor %xmm0, %xmm42150; SSSE3-NEXT: movdqa %xmm4, %xmm02151; SSSE3-NEXT: movdqa %xmm7, 48(%rsi)2152; SSSE3-NEXT: movdqa %xmm1, %xmm72153; SSSE3-NEXT: movdqa %xmm6, 32(%rsi)2154; SSSE3-NEXT: movdqa %xmm2, %xmm62155; SSSE3-NEXT: movdqa %xmm5, 16(%rsi)2156; SSSE3-NEXT: movdqa %xmm3, %xmm52157; SSSE3-NEXT: movdqa %xmm8, (%rsi)2158; SSSE3-NEXT: movdqa %xmm3, %xmm82159; SSSE3-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]2160; SSSE3-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]2161; SSSE3-NEXT: movdqa %xmm3, 192(%rdi)2162; SSSE3-NEXT: movdqa %xmm2, %xmm32163; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]2164; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]2165; SSSE3-NEXT: movdqa %xmm2, 128(%rdi)2166; SSSE3-NEXT: movdqa %xmm1, %xmm22167; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]2168; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]2169; SSSE3-NEXT: movdqa %xmm1, 64(%rdi)2170; SSSE3-NEXT: movdqa %xmm4, %xmm12171; SSSE3-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2172; SSSE3-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]2173; SSSE3-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3]2174; SSSE3-NEXT: movdqa %xmm4, (%rdi)2175; SSSE3-NEXT: movdqa %xmm8, %xmm42176; SSSE3-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0,0,1,1,2,2,3,3]2177; SSSE3-NEXT: pslld $31, %xmm82178; SSSE3-NEXT: psrad $31, %xmm82179; SSSE3-NEXT: movdqa %xmm8, 224(%rdi)2180; SSSE3-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]2181; SSSE3-NEXT: pslld $31, %xmm42182; SSSE3-NEXT: psrad $31, %xmm42183; SSSE3-NEXT: movdqa %xmm4, 240(%rdi)2184; SSSE3-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2185; SSSE3-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]2186; SSSE3-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]2187; SSSE3-NEXT: pslld $31, %xmm52188; SSSE3-NEXT: psrad $31, %xmm52189; SSSE3-NEXT: movdqa %xmm5, 208(%rdi)2190; SSSE3-NEXT: movdqa %xmm3, %xmm42191; SSSE3-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]2192; SSSE3-NEXT: pslld $31, %xmm32193; SSSE3-NEXT: psrad $31, %xmm32194; SSSE3-NEXT: movdqa %xmm3, 160(%rdi)2195; SSSE3-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]2196; SSSE3-NEXT: pslld $31, %xmm42197; SSSE3-NEXT: psrad $31, %xmm42198; SSSE3-NEXT: movdqa %xmm4, 176(%rdi)2199; SSSE3-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2200; SSSE3-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]2201; SSSE3-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4,4,5,5,6,6,7,7]2202; SSSE3-NEXT: pslld $31, %xmm62203; SSSE3-NEXT: psrad $31, %xmm62204; SSSE3-NEXT: movdqa %xmm6, 144(%rdi)2205; SSSE3-NEXT: movdqa %xmm2, %xmm32206; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]2207; SSSE3-NEXT: pslld $31, %xmm22208; SSSE3-NEXT: psrad $31, %xmm22209; SSSE3-NEXT: movdqa %xmm2, 96(%rdi)2210; SSSE3-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]2211; SSSE3-NEXT: pslld $31, %xmm32212; SSSE3-NEXT: psrad $31, %xmm32213; SSSE3-NEXT: movdqa %xmm3, 112(%rdi)2214; SSSE3-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]2215; SSSE3-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]2216; SSSE3-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4,4,5,5,6,6,7,7]2217; SSSE3-NEXT: pslld $31, %xmm72218; SSSE3-NEXT: psrad $31, %xmm72219; SSSE3-NEXT: movdqa %xmm7, 80(%rdi)2220; SSSE3-NEXT: movdqa %xmm1, %xmm22221; SSSE3-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]2222; SSSE3-NEXT: pslld $31, %xmm12223; SSSE3-NEXT: psrad $31, %xmm12224; SSSE3-NEXT: movdqa %xmm1, 32(%rdi)2225; SSSE3-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]2226; SSSE3-NEXT: pslld $31, %xmm22227; SSSE3-NEXT: psrad $31, %xmm22228; SSSE3-NEXT: movdqa %xmm2, 48(%rdi)2229; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]2230; SSSE3-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]2231; SSSE3-NEXT: pslld $31, %xmm02232; SSSE3-NEXT: psrad $31, %xmm02233; SSSE3-NEXT: movdqa %xmm0, 16(%rdi)2234; SSSE3-NEXT: retq2235;2236; SSE41-LABEL: smulo_v64i8:2237; SSE41: # %bb.0:2238; SSE41-NEXT: movq %rdi, %rax2239; SSE41-NEXT: pxor %xmm8, %xmm82240; SSE41-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15]2241; SSE41-NEXT: pxor %xmm9, %xmm92242; SSE41-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm3[8],xmm9[9],xmm3[9],xmm9[10],xmm3[10],xmm9[11],xmm3[11],xmm9[12],xmm3[12],xmm9[13],xmm3[13],xmm9[14],xmm3[14],xmm9[15],xmm3[15]2243; SSE41-NEXT: pmulhw %xmm8, %xmm92244; SSE41-NEXT: movdqa %xmm9, %xmm82245; SSE41-NEXT: psrlw $8, %xmm82246; SSE41-NEXT: pxor %xmm10, %xmm102247; SSE41-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm7[0],xmm10[1],xmm7[1],xmm10[2],xmm7[2],xmm10[3],xmm7[3],xmm10[4],xmm7[4],xmm10[5],xmm7[5],xmm10[6],xmm7[6],xmm10[7],xmm7[7]2248; SSE41-NEXT: pxor %xmm7, %xmm72249; SSE41-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3],xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]2250; SSE41-NEXT: pmulhw %xmm10, %xmm72251; SSE41-NEXT: movdqa %xmm7, %xmm102252; SSE41-NEXT: psrlw $8, %xmm102253; SSE41-NEXT: packuswb %xmm8, %xmm102254; SSE41-NEXT: pmovzxbw {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255]2255; SSE41-NEXT: pand %xmm8, %xmm92256; SSE41-NEXT: pand %xmm8, %xmm72257; SSE41-NEXT: packuswb %xmm9, %xmm72258; SSE41-NEXT: pxor %xmm3, %xmm32259; SSE41-NEXT: pcmpgtb %xmm7, %xmm32260; SSE41-NEXT: pcmpeqb %xmm10, %xmm32261; SSE41-NEXT: pxor %xmm9, %xmm92262; SSE41-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm6[8],xmm9[9],xmm6[9],xmm9[10],xmm6[10],xmm9[11],xmm6[11],xmm9[12],xmm6[12],xmm9[13],xmm6[13],xmm9[14],xmm6[14],xmm9[15],xmm6[15]2263; SSE41-NEXT: pxor %xmm10, %xmm102264; SSE41-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm2[8],xmm10[9],xmm2[9],xmm10[10],xmm2[10],xmm10[11],xmm2[11],xmm10[12],xmm2[12],xmm10[13],xmm2[13],xmm10[14],xmm2[14],xmm10[15],xmm2[15]2265; SSE41-NEXT: pmulhw %xmm9, %xmm102266; SSE41-NEXT: pxor %xmm9, %xmm92267; SSE41-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3],xmm9[4],xmm6[4],xmm9[5],xmm6[5],xmm9[6],xmm6[6],xmm9[7],xmm6[7]2268; SSE41-NEXT: pxor %xmm6, %xmm62269; SSE41-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]2270; SSE41-NEXT: movdqa %xmm10, %xmm22271; SSE41-NEXT: psrlw $8, %xmm22272; SSE41-NEXT: pmulhw %xmm9, %xmm62273; SSE41-NEXT: movdqa %xmm6, %xmm92274; SSE41-NEXT: psrlw $8, %xmm92275; SSE41-NEXT: packuswb %xmm2, %xmm92276; SSE41-NEXT: pand %xmm8, %xmm102277; SSE41-NEXT: pand %xmm8, %xmm62278; SSE41-NEXT: packuswb %xmm10, %xmm62279; SSE41-NEXT: pxor %xmm2, %xmm22280; SSE41-NEXT: pcmpgtb %xmm6, %xmm22281; SSE41-NEXT: pcmpeqb %xmm9, %xmm22282; SSE41-NEXT: pxor %xmm9, %xmm92283; SSE41-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm5[8],xmm9[9],xmm5[9],xmm9[10],xmm5[10],xmm9[11],xmm5[11],xmm9[12],xmm5[12],xmm9[13],xmm5[13],xmm9[14],xmm5[14],xmm9[15],xmm5[15]2284; SSE41-NEXT: pxor %xmm10, %xmm102285; SSE41-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm1[8],xmm10[9],xmm1[9],xmm10[10],xmm1[10],xmm10[11],xmm1[11],xmm10[12],xmm1[12],xmm10[13],xmm1[13],xmm10[14],xmm1[14],xmm10[15],xmm1[15]2286; SSE41-NEXT: pmulhw %xmm9, %xmm102287; SSE41-NEXT: pxor %xmm9, %xmm92288; SSE41-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3],xmm9[4],xmm5[4],xmm9[5],xmm5[5],xmm9[6],xmm5[6],xmm9[7],xmm5[7]2289; SSE41-NEXT: pxor %xmm5, %xmm52290; SSE41-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]2291; SSE41-NEXT: movdqa %xmm10, %xmm12292; SSE41-NEXT: psrlw $8, %xmm12293; SSE41-NEXT: pmulhw %xmm9, %xmm52294; SSE41-NEXT: movdqa %xmm5, %xmm92295; SSE41-NEXT: psrlw $8, %xmm92296; SSE41-NEXT: packuswb %xmm1, %xmm92297; SSE41-NEXT: pand %xmm8, %xmm102298; SSE41-NEXT: pand %xmm8, %xmm52299; SSE41-NEXT: packuswb %xmm10, %xmm52300; SSE41-NEXT: pxor %xmm1, %xmm12301; SSE41-NEXT: pcmpgtb %xmm5, %xmm12302; SSE41-NEXT: pcmpeqb %xmm9, %xmm12303; SSE41-NEXT: pxor %xmm9, %xmm92304; SSE41-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15]2305; SSE41-NEXT: pxor %xmm10, %xmm102306; SSE41-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm0[8],xmm10[9],xmm0[9],xmm10[10],xmm0[10],xmm10[11],xmm0[11],xmm10[12],xmm0[12],xmm10[13],xmm0[13],xmm10[14],xmm0[14],xmm10[15],xmm0[15]2307; SSE41-NEXT: pmulhw %xmm9, %xmm102308; SSE41-NEXT: pxor %xmm9, %xmm92309; SSE41-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3],xmm9[4],xmm4[4],xmm9[5],xmm4[5],xmm9[6],xmm4[6],xmm9[7],xmm4[7]2310; SSE41-NEXT: pxor %xmm4, %xmm42311; SSE41-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]2312; SSE41-NEXT: movdqa %xmm10, %xmm02313; SSE41-NEXT: psrlw $8, %xmm02314; SSE41-NEXT: pmulhw %xmm9, %xmm42315; SSE41-NEXT: movdqa %xmm4, %xmm92316; SSE41-NEXT: psrlw $8, %xmm92317; SSE41-NEXT: packuswb %xmm0, %xmm92318; SSE41-NEXT: pand %xmm8, %xmm102319; SSE41-NEXT: pand %xmm8, %xmm42320; SSE41-NEXT: packuswb %xmm10, %xmm42321; SSE41-NEXT: pxor %xmm0, %xmm02322; SSE41-NEXT: pcmpgtb %xmm4, %xmm02323; SSE41-NEXT: pcmpeqb %xmm9, %xmm02324; SSE41-NEXT: pcmpeqd %xmm8, %xmm82325; SSE41-NEXT: pxor %xmm8, %xmm32326; SSE41-NEXT: pxor %xmm8, %xmm22327; SSE41-NEXT: pxor %xmm8, %xmm12328; SSE41-NEXT: pxor %xmm8, %xmm02329; SSE41-NEXT: movdqa %xmm7, 48(%rsi)2330; SSE41-NEXT: movdqa %xmm6, 32(%rsi)2331; SSE41-NEXT: movdqa %xmm5, 16(%rsi)2332; SSE41-NEXT: movdqa %xmm4, (%rsi)2333; SSE41-NEXT: pmovsxbd %xmm3, %xmm42334; SSE41-NEXT: movdqa %xmm4, 192(%rdi)2335; SSE41-NEXT: pmovsxbd %xmm2, %xmm42336; SSE41-NEXT: movdqa %xmm4, 128(%rdi)2337; SSE41-NEXT: pmovsxbd %xmm1, %xmm42338; SSE41-NEXT: movdqa %xmm4, 64(%rdi)2339; SSE41-NEXT: pmovsxbd %xmm0, %xmm42340; SSE41-NEXT: movdqa %xmm4, (%rdi)2341; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]2342; SSE41-NEXT: pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero2343; SSE41-NEXT: pslld $31, %xmm42344; SSE41-NEXT: psrad $31, %xmm42345; SSE41-NEXT: movdqa %xmm4, 224(%rdi)2346; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm3[3,3,3,3]2347; SSE41-NEXT: pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero2348; SSE41-NEXT: pslld $31, %xmm42349; SSE41-NEXT: psrad $31, %xmm42350; SSE41-NEXT: movdqa %xmm4, 240(%rdi)2351; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,1,1]2352; SSE41-NEXT: pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero2353; SSE41-NEXT: pslld $31, %xmm32354; SSE41-NEXT: psrad $31, %xmm32355; SSE41-NEXT: movdqa %xmm3, 208(%rdi)2356; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]2357; SSE41-NEXT: pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero2358; SSE41-NEXT: pslld $31, %xmm32359; SSE41-NEXT: psrad $31, %xmm32360; SSE41-NEXT: movdqa %xmm3, 160(%rdi)2361; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[3,3,3,3]2362; SSE41-NEXT: pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero2363; SSE41-NEXT: pslld $31, %xmm32364; SSE41-NEXT: psrad $31, %xmm32365; SSE41-NEXT: movdqa %xmm3, 176(%rdi)2366; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]2367; SSE41-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero2368; SSE41-NEXT: pslld $31, %xmm22369; SSE41-NEXT: psrad $31, %xmm22370; SSE41-NEXT: movdqa %xmm2, 144(%rdi)2371; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]2372; SSE41-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero2373; SSE41-NEXT: pslld $31, %xmm22374; SSE41-NEXT: psrad $31, %xmm22375; SSE41-NEXT: movdqa %xmm2, 96(%rdi)2376; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]2377; SSE41-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero2378; SSE41-NEXT: pslld $31, %xmm22379; SSE41-NEXT: psrad $31, %xmm22380; SSE41-NEXT: movdqa %xmm2, 112(%rdi)2381; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]2382; SSE41-NEXT: pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero2383; SSE41-NEXT: pslld $31, %xmm12384; SSE41-NEXT: psrad $31, %xmm12385; SSE41-NEXT: movdqa %xmm1, 80(%rdi)2386; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]2387; SSE41-NEXT: pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero2388; SSE41-NEXT: pslld $31, %xmm12389; SSE41-NEXT: psrad $31, %xmm12390; SSE41-NEXT: movdqa %xmm1, 32(%rdi)2391; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]2392; SSE41-NEXT: pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero2393; SSE41-NEXT: pslld $31, %xmm12394; SSE41-NEXT: psrad $31, %xmm12395; SSE41-NEXT: movdqa %xmm1, 48(%rdi)2396; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]2397; SSE41-NEXT: pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero2398; SSE41-NEXT: pslld $31, %xmm02399; SSE41-NEXT: psrad $31, %xmm02400; SSE41-NEXT: movdqa %xmm0, 16(%rdi)2401; SSE41-NEXT: retq2402;2403; AVX1-LABEL: smulo_v64i8:2404; AVX1: # %bb.0:2405; AVX1-NEXT: movq %rdi, %rax2406; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm42407; AVX1-NEXT: vpxor %xmm5, %xmm5, %xmm52408; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm5[8],xmm4[8],xmm5[9],xmm4[9],xmm5[10],xmm4[10],xmm5[11],xmm4[11],xmm5[12],xmm4[12],xmm5[13],xmm4[13],xmm5[14],xmm4[14],xmm5[15],xmm4[15]2409; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm72410; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm8 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15]2411; AVX1-NEXT: vpmulhw %xmm6, %xmm8, %xmm62412; AVX1-NEXT: vpsrlw $8, %xmm6, %xmm82413; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]2414; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm7 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7]2415; AVX1-NEXT: vpmulhw %xmm4, %xmm7, %xmm42416; AVX1-NEXT: vpsrlw $8, %xmm4, %xmm72417; AVX1-NEXT: vpackuswb %xmm8, %xmm7, %xmm82418; AVX1-NEXT: vbroadcastss {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255]2419; AVX1-NEXT: vpand %xmm7, %xmm6, %xmm62420; AVX1-NEXT: vpand %xmm7, %xmm4, %xmm42421; AVX1-NEXT: vpackuswb %xmm6, %xmm4, %xmm42422; AVX1-NEXT: vpcmpgtb %xmm4, %xmm5, %xmm62423; AVX1-NEXT: vpcmpeqb %xmm6, %xmm8, %xmm82424; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]2425; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm9 = xmm5[8],xmm1[8],xmm5[9],xmm1[9],xmm5[10],xmm1[10],xmm5[11],xmm1[11],xmm5[12],xmm1[12],xmm5[13],xmm1[13],xmm5[14],xmm1[14],xmm5[15],xmm1[15]2426; AVX1-NEXT: vpmulhw %xmm6, %xmm9, %xmm62427; AVX1-NEXT: vpsrlw $8, %xmm6, %xmm92428; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]2429; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]2430; AVX1-NEXT: vpmulhw %xmm3, %xmm1, %xmm12431; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm32432; AVX1-NEXT: vpackuswb %xmm9, %xmm3, %xmm32433; AVX1-NEXT: vpand %xmm7, %xmm6, %xmm62434; AVX1-NEXT: vpand %xmm7, %xmm1, %xmm12435; AVX1-NEXT: vpackuswb %xmm6, %xmm1, %xmm12436; AVX1-NEXT: vpcmpgtb %xmm1, %xmm5, %xmm62437; AVX1-NEXT: vpcmpeqb %xmm3, %xmm6, %xmm32438; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm62439; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm9 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]2440; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm102441; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm11 = xmm5[8],xmm10[8],xmm5[9],xmm10[9],xmm5[10],xmm10[10],xmm5[11],xmm10[11],xmm5[12],xmm10[12],xmm5[13],xmm10[13],xmm5[14],xmm10[14],xmm5[15],xmm10[15]2442; AVX1-NEXT: vpmulhw %xmm9, %xmm11, %xmm92443; AVX1-NEXT: vpsrlw $8, %xmm9, %xmm112444; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm6 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3],xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]2445; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm10 = xmm5[0],xmm10[0],xmm5[1],xmm10[1],xmm5[2],xmm10[2],xmm5[3],xmm10[3],xmm5[4],xmm10[4],xmm5[5],xmm10[5],xmm5[6],xmm10[6],xmm5[7],xmm10[7]2446; AVX1-NEXT: vpmulhw %xmm6, %xmm10, %xmm62447; AVX1-NEXT: vpsrlw $8, %xmm6, %xmm102448; AVX1-NEXT: vpackuswb %xmm11, %xmm10, %xmm102449; AVX1-NEXT: vpand %xmm7, %xmm9, %xmm92450; AVX1-NEXT: vpand %xmm7, %xmm6, %xmm62451; AVX1-NEXT: vpackuswb %xmm9, %xmm6, %xmm62452; AVX1-NEXT: vpcmpgtb %xmm6, %xmm5, %xmm92453; AVX1-NEXT: vpcmpeqb %xmm10, %xmm9, %xmm92454; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm10 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]2455; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm11 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]2456; AVX1-NEXT: vpmulhw %xmm10, %xmm11, %xmm102457; AVX1-NEXT: vpsrlw $8, %xmm10, %xmm112458; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]2459; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]2460; AVX1-NEXT: vpmulhw %xmm2, %xmm0, %xmm02461; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm22462; AVX1-NEXT: vpackuswb %xmm11, %xmm2, %xmm22463; AVX1-NEXT: vpand %xmm7, %xmm10, %xmm102464; AVX1-NEXT: vpand %xmm7, %xmm0, %xmm02465; AVX1-NEXT: vpackuswb %xmm10, %xmm0, %xmm72466; AVX1-NEXT: vpcmpgtb %xmm7, %xmm5, %xmm02467; AVX1-NEXT: vpcmpeqb %xmm2, %xmm0, %xmm02468; AVX1-NEXT: vpcmpeqd %xmm10, %xmm10, %xmm102469; AVX1-NEXT: vpxor %xmm10, %xmm8, %xmm52470; AVX1-NEXT: vpxor %xmm3, %xmm10, %xmm32471; AVX1-NEXT: vpxor %xmm10, %xmm9, %xmm22472; AVX1-NEXT: vpxor %xmm0, %xmm10, %xmm02473; AVX1-NEXT: vmovdqa %xmm4, 48(%rsi)2474; AVX1-NEXT: vmovdqa %xmm1, 32(%rsi)2475; AVX1-NEXT: vmovdqa %xmm6, 16(%rsi)2476; AVX1-NEXT: vmovdqa %xmm7, (%rsi)2477; AVX1-NEXT: vpmovsxbd %xmm5, %xmm12478; AVX1-NEXT: vmovdqa %xmm1, 192(%rdi)2479; AVX1-NEXT: vpmovsxbd %xmm3, %xmm12480; AVX1-NEXT: vmovdqa %xmm1, 128(%rdi)2481; AVX1-NEXT: vpmovsxbd %xmm2, %xmm12482; AVX1-NEXT: vmovdqa %xmm1, 64(%rdi)2483; AVX1-NEXT: vpmovsxbd %xmm0, %xmm12484; AVX1-NEXT: vmovdqa %xmm1, (%rdi)2485; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm5[2,3,2,3]2486; AVX1-NEXT: vpmovsxbd %xmm1, %xmm12487; AVX1-NEXT: vmovdqa %xmm1, 224(%rdi)2488; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm5[3,3,3,3]2489; AVX1-NEXT: vpmovsxbd %xmm1, %xmm12490; AVX1-NEXT: vmovdqa %xmm1, 240(%rdi)2491; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm5[1,1,1,1]2492; AVX1-NEXT: vpmovsxbd %xmm1, %xmm12493; AVX1-NEXT: vmovdqa %xmm1, 208(%rdi)2494; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]2495; AVX1-NEXT: vpmovsxbd %xmm1, %xmm12496; AVX1-NEXT: vmovdqa %xmm1, 160(%rdi)2497; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[3,3,3,3]2498; AVX1-NEXT: vpmovsxbd %xmm1, %xmm12499; AVX1-NEXT: vmovdqa %xmm1, 176(%rdi)2500; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]2501; AVX1-NEXT: vpmovsxbd %xmm1, %xmm12502; AVX1-NEXT: vmovdqa %xmm1, 144(%rdi)2503; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]2504; AVX1-NEXT: vpmovsxbd %xmm1, %xmm12505; AVX1-NEXT: vmovdqa %xmm1, 96(%rdi)2506; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]2507; AVX1-NEXT: vpmovsxbd %xmm1, %xmm12508; AVX1-NEXT: vmovdqa %xmm1, 112(%rdi)2509; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]2510; AVX1-NEXT: vpmovsxbd %xmm1, %xmm12511; AVX1-NEXT: vmovdqa %xmm1, 80(%rdi)2512; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]2513; AVX1-NEXT: vpmovsxbd %xmm1, %xmm12514; AVX1-NEXT: vmovdqa %xmm1, 32(%rdi)2515; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]2516; AVX1-NEXT: vpmovsxbd %xmm1, %xmm12517; AVX1-NEXT: vmovdqa %xmm1, 48(%rdi)2518; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]2519; AVX1-NEXT: vpmovsxbd %xmm0, %xmm02520; AVX1-NEXT: vmovdqa %xmm0, 16(%rdi)2521; AVX1-NEXT: vzeroupper2522; AVX1-NEXT: retq2523;2524; AVX2-LABEL: smulo_v64i8:2525; AVX2: # %bb.0:2526; AVX2-NEXT: movq %rdi, %rax2527; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm42528; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm5 = ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15],ymm4[24],ymm3[24],ymm4[25],ymm3[25],ymm4[26],ymm3[26],ymm4[27],ymm3[27],ymm4[28],ymm3[28],ymm4[29],ymm3[29],ymm4[30],ymm3[30],ymm4[31],ymm3[31]2529; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm6 = ymm4[8],ymm1[8],ymm4[9],ymm1[9],ymm4[10],ymm1[10],ymm4[11],ymm1[11],ymm4[12],ymm1[12],ymm4[13],ymm1[13],ymm4[14],ymm1[14],ymm4[15],ymm1[15],ymm4[24],ymm1[24],ymm4[25],ymm1[25],ymm4[26],ymm1[26],ymm4[27],ymm1[27],ymm4[28],ymm1[28],ymm4[29],ymm1[29],ymm4[30],ymm1[30],ymm4[31],ymm1[31]2530; AVX2-NEXT: vpmulhw %ymm5, %ymm6, %ymm52531; AVX2-NEXT: vpsrlw $8, %ymm5, %ymm62532; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[16],ymm3[16],ymm4[17],ymm3[17],ymm4[18],ymm3[18],ymm4[19],ymm3[19],ymm4[20],ymm3[20],ymm4[21],ymm3[21],ymm4[22],ymm3[22],ymm4[23],ymm3[23]2533; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm4[0],ymm1[0],ymm4[1],ymm1[1],ymm4[2],ymm1[2],ymm4[3],ymm1[3],ymm4[4],ymm1[4],ymm4[5],ymm1[5],ymm4[6],ymm1[6],ymm4[7],ymm1[7],ymm4[16],ymm1[16],ymm4[17],ymm1[17],ymm4[18],ymm1[18],ymm4[19],ymm1[19],ymm4[20],ymm1[20],ymm4[21],ymm1[21],ymm4[22],ymm1[22],ymm4[23],ymm1[23]2534; AVX2-NEXT: vpmulhw %ymm3, %ymm1, %ymm12535; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm32536; AVX2-NEXT: vpackuswb %ymm6, %ymm3, %ymm32537; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm6 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]2538; AVX2-NEXT: vpand %ymm6, %ymm5, %ymm52539; AVX2-NEXT: vpand %ymm6, %ymm1, %ymm12540; AVX2-NEXT: vpackuswb %ymm5, %ymm1, %ymm12541; AVX2-NEXT: vpcmpgtb %ymm1, %ymm4, %ymm52542; AVX2-NEXT: vpcmpeqb %ymm3, %ymm5, %ymm32543; AVX2-NEXT: vpcmpeqd %ymm5, %ymm5, %ymm52544; AVX2-NEXT: vpxor %ymm5, %ymm3, %ymm32545; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm7 = ymm4[8],ymm2[8],ymm4[9],ymm2[9],ymm4[10],ymm2[10],ymm4[11],ymm2[11],ymm4[12],ymm2[12],ymm4[13],ymm2[13],ymm4[14],ymm2[14],ymm4[15],ymm2[15],ymm4[24],ymm2[24],ymm4[25],ymm2[25],ymm4[26],ymm2[26],ymm4[27],ymm2[27],ymm4[28],ymm2[28],ymm4[29],ymm2[29],ymm4[30],ymm2[30],ymm4[31],ymm2[31]2546; AVX2-NEXT: vpunpckhbw {{.*#+}} ymm8 = ymm4[8],ymm0[8],ymm4[9],ymm0[9],ymm4[10],ymm0[10],ymm4[11],ymm0[11],ymm4[12],ymm0[12],ymm4[13],ymm0[13],ymm4[14],ymm0[14],ymm4[15],ymm0[15],ymm4[24],ymm0[24],ymm4[25],ymm0[25],ymm4[26],ymm0[26],ymm4[27],ymm0[27],ymm4[28],ymm0[28],ymm4[29],ymm0[29],ymm4[30],ymm0[30],ymm4[31],ymm0[31]2547; AVX2-NEXT: vpmulhw %ymm7, %ymm8, %ymm72548; AVX2-NEXT: vpsrlw $8, %ymm7, %ymm82549; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[2],ymm2[2],ymm4[3],ymm2[3],ymm4[4],ymm2[4],ymm4[5],ymm2[5],ymm4[6],ymm2[6],ymm4[7],ymm2[7],ymm4[16],ymm2[16],ymm4[17],ymm2[17],ymm4[18],ymm2[18],ymm4[19],ymm2[19],ymm4[20],ymm2[20],ymm4[21],ymm2[21],ymm4[22],ymm2[22],ymm4[23],ymm2[23]2550; AVX2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm4[0],ymm0[0],ymm4[1],ymm0[1],ymm4[2],ymm0[2],ymm4[3],ymm0[3],ymm4[4],ymm0[4],ymm4[5],ymm0[5],ymm4[6],ymm0[6],ymm4[7],ymm0[7],ymm4[16],ymm0[16],ymm4[17],ymm0[17],ymm4[18],ymm0[18],ymm4[19],ymm0[19],ymm4[20],ymm0[20],ymm4[21],ymm0[21],ymm4[22],ymm0[22],ymm4[23],ymm0[23]2551; AVX2-NEXT: vpmulhw %ymm2, %ymm0, %ymm02552; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm22553; AVX2-NEXT: vpackuswb %ymm8, %ymm2, %ymm22554; AVX2-NEXT: vpand %ymm6, %ymm7, %ymm72555; AVX2-NEXT: vpand %ymm6, %ymm0, %ymm02556; AVX2-NEXT: vpackuswb %ymm7, %ymm0, %ymm02557; AVX2-NEXT: vpcmpgtb %ymm0, %ymm4, %ymm42558; AVX2-NEXT: vpcmpeqb %ymm2, %ymm4, %ymm22559; AVX2-NEXT: vpxor %ymm5, %ymm2, %ymm22560; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]2561; AVX2-NEXT: vpmovsxbd %xmm4, %ymm42562; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm52563; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]2564; AVX2-NEXT: vpmovsxbd %xmm6, %ymm62565; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[2,3,2,3]2566; AVX2-NEXT: vpmovsxbd %xmm7, %ymm72567; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm82568; AVX2-NEXT: vpshufd {{.*#+}} xmm9 = xmm8[2,3,2,3]2569; AVX2-NEXT: vpmovsxbd %xmm9, %ymm92570; AVX2-NEXT: vpmovsxbd %xmm2, %ymm22571; AVX2-NEXT: vpmovsxbd %xmm5, %ymm52572; AVX2-NEXT: vpmovsxbd %xmm3, %ymm32573; AVX2-NEXT: vpmovsxbd %xmm8, %ymm82574; AVX2-NEXT: vmovdqa %ymm1, 32(%rsi)2575; AVX2-NEXT: vmovdqa %ymm0, (%rsi)2576; AVX2-NEXT: vmovdqa %ymm8, 192(%rdi)2577; AVX2-NEXT: vmovdqa %ymm3, 128(%rdi)2578; AVX2-NEXT: vmovdqa %ymm5, 64(%rdi)2579; AVX2-NEXT: vmovdqa %ymm2, (%rdi)2580; AVX2-NEXT: vmovdqa %ymm9, 224(%rdi)2581; AVX2-NEXT: vmovdqa %ymm7, 160(%rdi)2582; AVX2-NEXT: vmovdqa %ymm6, 96(%rdi)2583; AVX2-NEXT: vmovdqa %ymm4, 32(%rdi)2584; AVX2-NEXT: vzeroupper2585; AVX2-NEXT: retq2586;2587; AVX512F-LABEL: smulo_v64i8:2588; AVX512F: # %bb.0:2589; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm22590; AVX512F-NEXT: vextracti128 $1, %ymm2, %xmm32591; AVX512F-NEXT: vpmovsxbw %xmm3, %ymm32592; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm52593; AVX512F-NEXT: vextracti128 $1, %ymm5, %xmm42594; AVX512F-NEXT: vpmovsxbw %xmm4, %ymm42595; AVX512F-NEXT: vpmullw %ymm3, %ymm4, %ymm42596; AVX512F-NEXT: vpsraw $8, %ymm4, %ymm32597; AVX512F-NEXT: vpmovsxwd %ymm3, %zmm32598; AVX512F-NEXT: vpsllw $8, %ymm4, %ymm62599; AVX512F-NEXT: vpsraw $15, %ymm6, %ymm62600; AVX512F-NEXT: vpmovsxwd %ymm6, %zmm62601; AVX512F-NEXT: vpcmpneqd %zmm3, %zmm6, %k12602; AVX512F-NEXT: vpmovsxbw %xmm2, %ymm22603; AVX512F-NEXT: vpmovsxbw %xmm5, %ymm32604; AVX512F-NEXT: vpmullw %ymm2, %ymm3, %ymm52605; AVX512F-NEXT: vpsraw $8, %ymm5, %ymm22606; AVX512F-NEXT: vpmovsxwd %ymm2, %zmm22607; AVX512F-NEXT: vpsllw $8, %ymm5, %ymm32608; AVX512F-NEXT: vpsraw $15, %ymm3, %ymm32609; AVX512F-NEXT: vpmovsxwd %ymm3, %zmm32610; AVX512F-NEXT: vpcmpneqd %zmm2, %zmm3, %k22611; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm22612; AVX512F-NEXT: vpmovsxbw %xmm2, %ymm22613; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm32614; AVX512F-NEXT: vpmovsxbw %xmm3, %ymm32615; AVX512F-NEXT: vpmullw %ymm2, %ymm3, %ymm62616; AVX512F-NEXT: vpsraw $8, %ymm6, %ymm22617; AVX512F-NEXT: vpmovsxwd %ymm2, %zmm22618; AVX512F-NEXT: vpsllw $8, %ymm6, %ymm32619; AVX512F-NEXT: vpsraw $15, %ymm3, %ymm32620; AVX512F-NEXT: vpmovsxwd %ymm3, %zmm32621; AVX512F-NEXT: vpcmpneqd %zmm2, %zmm3, %k32622; AVX512F-NEXT: vpmovsxbw %xmm1, %ymm12623; AVX512F-NEXT: vpmovsxbw %xmm0, %ymm02624; AVX512F-NEXT: vpmullw %ymm1, %ymm0, %ymm72625; AVX512F-NEXT: vpsraw $8, %ymm7, %ymm02626; AVX512F-NEXT: vpmovsxwd %ymm0, %zmm02627; AVX512F-NEXT: vpsllw $8, %ymm7, %ymm12628; AVX512F-NEXT: vpsraw $15, %ymm1, %ymm12629; AVX512F-NEXT: vpmovsxwd %ymm1, %zmm12630; AVX512F-NEXT: vpcmpneqd %zmm0, %zmm1, %k42631; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 {%k4} {z} = -12632; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k3} {z} = -12633; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k2} {z} = -12634; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k1} {z} = -12635; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero2636; AVX512F-NEXT: vpmovdb %zmm4, 48(%rdi)2637; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm4 = ymm5[0],zero,ymm5[1],zero,ymm5[2],zero,ymm5[3],zero,ymm5[4],zero,ymm5[5],zero,ymm5[6],zero,ymm5[7],zero,ymm5[8],zero,ymm5[9],zero,ymm5[10],zero,ymm5[11],zero,ymm5[12],zero,ymm5[13],zero,ymm5[14],zero,ymm5[15],zero2638; AVX512F-NEXT: vpmovdb %zmm4, 32(%rdi)2639; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm4 = ymm6[0],zero,ymm6[1],zero,ymm6[2],zero,ymm6[3],zero,ymm6[4],zero,ymm6[5],zero,ymm6[6],zero,ymm6[7],zero,ymm6[8],zero,ymm6[9],zero,ymm6[10],zero,ymm6[11],zero,ymm6[12],zero,ymm6[13],zero,ymm6[14],zero,ymm6[15],zero2640; AVX512F-NEXT: vpmovdb %zmm4, 16(%rdi)2641; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm4 = ymm7[0],zero,ymm7[1],zero,ymm7[2],zero,ymm7[3],zero,ymm7[4],zero,ymm7[5],zero,ymm7[6],zero,ymm7[7],zero,ymm7[8],zero,ymm7[9],zero,ymm7[10],zero,ymm7[11],zero,ymm7[12],zero,ymm7[13],zero,ymm7[14],zero,ymm7[15],zero2642; AVX512F-NEXT: vpmovdb %zmm4, (%rdi)2643; AVX512F-NEXT: retq2644;2645; AVX512BW-LABEL: smulo_v64i8:2646; AVX512BW: # %bb.0:2647; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm22648; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm2[8],zmm1[8],zmm2[9],zmm1[9],zmm2[10],zmm1[10],zmm2[11],zmm1[11],zmm2[12],zmm1[12],zmm2[13],zmm1[13],zmm2[14],zmm1[14],zmm2[15],zmm1[15],zmm2[24],zmm1[24],zmm2[25],zmm1[25],zmm2[26],zmm1[26],zmm2[27],zmm1[27],zmm2[28],zmm1[28],zmm2[29],zmm1[29],zmm2[30],zmm1[30],zmm2[31],zmm1[31],zmm2[40],zmm1[40],zmm2[41],zmm1[41],zmm2[42],zmm1[42],zmm2[43],zmm1[43],zmm2[44],zmm1[44],zmm2[45],zmm1[45],zmm2[46],zmm1[46],zmm2[47],zmm1[47],zmm2[56],zmm1[56],zmm2[57],zmm1[57],zmm2[58],zmm1[58],zmm2[59],zmm1[59],zmm2[60],zmm1[60],zmm2[61],zmm1[61],zmm2[62],zmm1[62],zmm2[63],zmm1[63]2649; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm2[8],zmm0[8],zmm2[9],zmm0[9],zmm2[10],zmm0[10],zmm2[11],zmm0[11],zmm2[12],zmm0[12],zmm2[13],zmm0[13],zmm2[14],zmm0[14],zmm2[15],zmm0[15],zmm2[24],zmm0[24],zmm2[25],zmm0[25],zmm2[26],zmm0[26],zmm2[27],zmm0[27],zmm2[28],zmm0[28],zmm2[29],zmm0[29],zmm2[30],zmm0[30],zmm2[31],zmm0[31],zmm2[40],zmm0[40],zmm2[41],zmm0[41],zmm2[42],zmm0[42],zmm2[43],zmm0[43],zmm2[44],zmm0[44],zmm2[45],zmm0[45],zmm2[46],zmm0[46],zmm2[47],zmm0[47],zmm2[56],zmm0[56],zmm2[57],zmm0[57],zmm2[58],zmm0[58],zmm2[59],zmm0[59],zmm2[60],zmm0[60],zmm2[61],zmm0[61],zmm2[62],zmm0[62],zmm2[63],zmm0[63]2650; AVX512BW-NEXT: vpmulhw %zmm3, %zmm4, %zmm32651; AVX512BW-NEXT: vpsrlw $8, %zmm3, %zmm42652; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm1 = zmm2[0],zmm1[0],zmm2[1],zmm1[1],zmm2[2],zmm1[2],zmm2[3],zmm1[3],zmm2[4],zmm1[4],zmm2[5],zmm1[5],zmm2[6],zmm1[6],zmm2[7],zmm1[7],zmm2[16],zmm1[16],zmm2[17],zmm1[17],zmm2[18],zmm1[18],zmm2[19],zmm1[19],zmm2[20],zmm1[20],zmm2[21],zmm1[21],zmm2[22],zmm1[22],zmm2[23],zmm1[23],zmm2[32],zmm1[32],zmm2[33],zmm1[33],zmm2[34],zmm1[34],zmm2[35],zmm1[35],zmm2[36],zmm1[36],zmm2[37],zmm1[37],zmm2[38],zmm1[38],zmm2[39],zmm1[39],zmm2[48],zmm1[48],zmm2[49],zmm1[49],zmm2[50],zmm1[50],zmm2[51],zmm1[51],zmm2[52],zmm1[52],zmm2[53],zmm1[53],zmm2[54],zmm1[54],zmm2[55],zmm1[55]2653; AVX512BW-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]2654; AVX512BW-NEXT: vpmulhw %zmm1, %zmm0, %zmm02655; AVX512BW-NEXT: vpsrlw $8, %zmm0, %zmm12656; AVX512BW-NEXT: vpackuswb %zmm4, %zmm1, %zmm12657; AVX512BW-NEXT: vpbroadcastw {{.*#+}} zmm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]2658; AVX512BW-NEXT: vpandq %zmm2, %zmm3, %zmm32659; AVX512BW-NEXT: vpandq %zmm2, %zmm0, %zmm02660; AVX512BW-NEXT: vpackuswb %zmm3, %zmm0, %zmm42661; AVX512BW-NEXT: vpmovb2m %zmm4, %k02662; AVX512BW-NEXT: vpmovm2b %k0, %zmm02663; AVX512BW-NEXT: vpcmpneqb %zmm1, %zmm0, %k12664; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -12665; AVX512BW-NEXT: kshiftrq $16, %k1, %k22666; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 {%k2} {z} = -12667; AVX512BW-NEXT: kshiftrq $32, %k1, %k22668; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm2 {%k2} {z} = -12669; AVX512BW-NEXT: kshiftrq $48, %k1, %k12670; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm3 {%k1} {z} = -12671; AVX512BW-NEXT: vmovdqa64 %zmm4, (%rdi)2672; AVX512BW-NEXT: retq2673 %t = call {<64 x i8>, <64 x i1>} @llvm.smul.with.overflow.v64i8(<64 x i8> %a0, <64 x i8> %a1)2674 %val = extractvalue {<64 x i8>, <64 x i1>} %t, 02675 %obit = extractvalue {<64 x i8>, <64 x i1>} %t, 12676 %res = sext <64 x i1> %obit to <64 x i32>2677 store <64 x i8> %val, ptr %p22678 ret <64 x i32> %res2679}2680 2681define <8 x i32> @smulo_v8i16(<8 x i16> %a0, <8 x i16> %a1, ptr %p2) nounwind {2682; SSE2-LABEL: smulo_v8i16:2683; SSE2: # %bb.0:2684; SSE2-NEXT: movdqa %xmm0, %xmm22685; SSE2-NEXT: pmulhw %xmm1, %xmm22686; SSE2-NEXT: pmullw %xmm1, %xmm02687; SSE2-NEXT: movdqa %xmm0, (%rdi)2688; SSE2-NEXT: psraw $15, %xmm02689; SSE2-NEXT: pcmpeqw %xmm2, %xmm02690; SSE2-NEXT: pcmpeqd %xmm1, %xmm12691; SSE2-NEXT: pxor %xmm0, %xmm12692; SSE2-NEXT: movdqa %xmm1, %xmm02693; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]2694; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]2695; SSE2-NEXT: pslld $31, %xmm12696; SSE2-NEXT: psrad $31, %xmm12697; SSE2-NEXT: retq2698;2699; SSSE3-LABEL: smulo_v8i16:2700; SSSE3: # %bb.0:2701; SSSE3-NEXT: movdqa %xmm0, %xmm22702; SSSE3-NEXT: pmulhw %xmm1, %xmm22703; SSSE3-NEXT: pmullw %xmm1, %xmm02704; SSSE3-NEXT: movdqa %xmm0, (%rdi)2705; SSSE3-NEXT: psraw $15, %xmm02706; SSSE3-NEXT: pcmpeqw %xmm2, %xmm02707; SSSE3-NEXT: pcmpeqd %xmm1, %xmm12708; SSSE3-NEXT: pxor %xmm0, %xmm12709; SSSE3-NEXT: movdqa %xmm1, %xmm02710; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]2711; SSSE3-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]2712; SSSE3-NEXT: pslld $31, %xmm12713; SSSE3-NEXT: psrad $31, %xmm12714; SSSE3-NEXT: retq2715;2716; SSE41-LABEL: smulo_v8i16:2717; SSE41: # %bb.0:2718; SSE41-NEXT: movdqa %xmm0, %xmm22719; SSE41-NEXT: pmulhw %xmm1, %xmm22720; SSE41-NEXT: pmullw %xmm1, %xmm02721; SSE41-NEXT: movdqa %xmm0, (%rdi)2722; SSE41-NEXT: psraw $15, %xmm02723; SSE41-NEXT: pcmpeqw %xmm2, %xmm02724; SSE41-NEXT: pcmpeqd %xmm1, %xmm12725; SSE41-NEXT: pxor %xmm0, %xmm12726; SSE41-NEXT: pmovsxwd %xmm1, %xmm02727; SSE41-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]2728; SSE41-NEXT: pslld $31, %xmm12729; SSE41-NEXT: psrad $31, %xmm12730; SSE41-NEXT: retq2731;2732; AVX1-LABEL: smulo_v8i16:2733; AVX1: # %bb.0:2734; AVX1-NEXT: vpmulhw %xmm1, %xmm0, %xmm22735; AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm12736; AVX1-NEXT: vpsraw $15, %xmm1, %xmm02737; AVX1-NEXT: vpcmpeqw %xmm0, %xmm2, %xmm02738; AVX1-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm22739; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm02740; AVX1-NEXT: vpmovsxwd %xmm0, %xmm22741; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]2742; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm02743; AVX1-NEXT: vmovdqa %xmm1, (%rdi)2744; AVX1-NEXT: retq2745;2746; AVX2-LABEL: smulo_v8i16:2747; AVX2: # %bb.0:2748; AVX2-NEXT: vpmulhw %xmm1, %xmm0, %xmm22749; AVX2-NEXT: vpmullw %xmm1, %xmm0, %xmm12750; AVX2-NEXT: vpsraw $15, %xmm1, %xmm02751; AVX2-NEXT: vpcmpeqw %xmm0, %xmm2, %xmm02752; AVX2-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm22753; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm02754; AVX2-NEXT: vpmovsxwd %xmm0, %ymm02755; AVX2-NEXT: vmovdqa %xmm1, (%rdi)2756; AVX2-NEXT: retq2757;2758; AVX512F-LABEL: smulo_v8i16:2759; AVX512F: # %bb.0:2760; AVX512F-NEXT: vpmulhw %xmm1, %xmm0, %xmm22761; AVX512F-NEXT: vpmullw %xmm1, %xmm0, %xmm12762; AVX512F-NEXT: vpsraw $15, %xmm1, %xmm02763; AVX512F-NEXT: vpcmpeqw %xmm0, %xmm2, %xmm02764; AVX512F-NEXT: vpternlogq {{.*#+}} xmm0 = ~xmm02765; AVX512F-NEXT: vpmovsxwd %xmm0, %ymm02766; AVX512F-NEXT: vptestmd %ymm0, %ymm0, %k12767; AVX512F-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm02768; AVX512F-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z}2769; AVX512F-NEXT: vmovdqa %xmm1, (%rdi)2770; AVX512F-NEXT: retq2771;2772; AVX512BW-LABEL: smulo_v8i16:2773; AVX512BW: # %bb.0:2774; AVX512BW-NEXT: vpmulhw %xmm1, %xmm0, %xmm22775; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm12776; AVX512BW-NEXT: vpsraw $15, %xmm1, %xmm02777; AVX512BW-NEXT: vpcmpneqw %xmm0, %xmm2, %k12778; AVX512BW-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm02779; AVX512BW-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z}2780; AVX512BW-NEXT: vmovdqa %xmm1, (%rdi)2781; AVX512BW-NEXT: retq2782 %t = call {<8 x i16>, <8 x i1>} @llvm.smul.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)2783 %val = extractvalue {<8 x i16>, <8 x i1>} %t, 02784 %obit = extractvalue {<8 x i16>, <8 x i1>} %t, 12785 %res = sext <8 x i1> %obit to <8 x i32>2786 store <8 x i16> %val, ptr %p22787 ret <8 x i32> %res2788}2789 2790define <2 x i32> @smulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, ptr %p2) nounwind {2791; SSE2-LABEL: smulo_v2i64:2792; SSE2: # %bb.0:2793; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]2794; SSE2-NEXT: movq %xmm2, %rax2795; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]2796; SSE2-NEXT: movq %xmm2, %rcx2797; SSE2-NEXT: movq %xmm1, %rdx2798; SSE2-NEXT: movq %xmm0, %rsi2799; SSE2-NEXT: xorl %r8d, %r8d2800; SSE2-NEXT: imulq %rdx, %rsi2801; SSE2-NEXT: movq $-1, %rdx2802; SSE2-NEXT: movl $0, %r9d2803; SSE2-NEXT: cmovoq %rdx, %r92804; SSE2-NEXT: movq %rsi, %xmm12805; SSE2-NEXT: imulq %rax, %rcx2806; SSE2-NEXT: movq %rcx, %xmm02807; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]2808; SSE2-NEXT: movq %r9, %xmm02809; SSE2-NEXT: cmovoq %rdx, %r82810; SSE2-NEXT: movq %r8, %xmm22811; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]2812; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]2813; SSE2-NEXT: movdqa %xmm1, (%rdi)2814; SSE2-NEXT: retq2815;2816; SSSE3-LABEL: smulo_v2i64:2817; SSSE3: # %bb.0:2818; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]2819; SSSE3-NEXT: movq %xmm2, %rax2820; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]2821; SSSE3-NEXT: movq %xmm2, %rcx2822; SSSE3-NEXT: movq %xmm1, %rdx2823; SSSE3-NEXT: movq %xmm0, %rsi2824; SSSE3-NEXT: xorl %r8d, %r8d2825; SSSE3-NEXT: imulq %rdx, %rsi2826; SSSE3-NEXT: movq $-1, %rdx2827; SSSE3-NEXT: movl $0, %r9d2828; SSSE3-NEXT: cmovoq %rdx, %r92829; SSSE3-NEXT: movq %rsi, %xmm12830; SSSE3-NEXT: imulq %rax, %rcx2831; SSSE3-NEXT: movq %rcx, %xmm02832; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]2833; SSSE3-NEXT: movq %r9, %xmm02834; SSSE3-NEXT: cmovoq %rdx, %r82835; SSSE3-NEXT: movq %r8, %xmm22836; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]2837; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]2838; SSSE3-NEXT: movdqa %xmm1, (%rdi)2839; SSSE3-NEXT: retq2840;2841; SSE41-LABEL: smulo_v2i64:2842; SSE41: # %bb.0:2843; SSE41-NEXT: movq %xmm1, %rax2844; SSE41-NEXT: movq %xmm0, %rcx2845; SSE41-NEXT: pextrq $1, %xmm1, %rdx2846; SSE41-NEXT: pextrq $1, %xmm0, %rsi2847; SSE41-NEXT: xorl %r8d, %r8d2848; SSE41-NEXT: imulq %rdx, %rsi2849; SSE41-NEXT: movq $-1, %rdx2850; SSE41-NEXT: movl $0, %r9d2851; SSE41-NEXT: cmovoq %rdx, %r92852; SSE41-NEXT: movq %rsi, %xmm02853; SSE41-NEXT: imulq %rax, %rcx2854; SSE41-NEXT: movq %rcx, %xmm12855; SSE41-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]2856; SSE41-NEXT: movq %r9, %xmm02857; SSE41-NEXT: cmovoq %rdx, %r82858; SSE41-NEXT: movq %r8, %xmm22859; SSE41-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]2860; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]2861; SSE41-NEXT: movdqa %xmm1, (%rdi)2862; SSE41-NEXT: retq2863;2864; AVX-LABEL: smulo_v2i64:2865; AVX: # %bb.0:2866; AVX-NEXT: vmovq %xmm1, %rax2867; AVX-NEXT: vmovq %xmm0, %rcx2868; AVX-NEXT: vpextrq $1, %xmm1, %rdx2869; AVX-NEXT: vpextrq $1, %xmm0, %rsi2870; AVX-NEXT: xorl %r8d, %r8d2871; AVX-NEXT: imulq %rdx, %rsi2872; AVX-NEXT: movq $-1, %rdx2873; AVX-NEXT: movl $0, %r9d2874; AVX-NEXT: cmovoq %rdx, %r92875; AVX-NEXT: vmovq %rsi, %xmm02876; AVX-NEXT: imulq %rax, %rcx2877; AVX-NEXT: vmovq %rcx, %xmm12878; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]2879; AVX-NEXT: vmovq %r9, %xmm02880; AVX-NEXT: cmovoq %rdx, %r82881; AVX-NEXT: vmovq %r8, %xmm22882; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm2[0],xmm0[0]2883; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]2884; AVX-NEXT: vmovdqa %xmm1, (%rdi)2885; AVX-NEXT: retq2886;2887; AVX512F-LABEL: smulo_v2i64:2888; AVX512F: # %bb.0:2889; AVX512F-NEXT: vmovq %xmm1, %rax2890; AVX512F-NEXT: vmovq %xmm0, %rcx2891; AVX512F-NEXT: vpextrq $1, %xmm1, %rdx2892; AVX512F-NEXT: vpextrq $1, %xmm0, %rsi2893; AVX512F-NEXT: imulq %rdx, %rsi2894; AVX512F-NEXT: seto %dl2895; AVX512F-NEXT: vmovq %rsi, %xmm02896; AVX512F-NEXT: imulq %rax, %rcx2897; AVX512F-NEXT: vmovq %rcx, %xmm12898; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]2899; AVX512F-NEXT: seto %al2900; AVX512F-NEXT: andl $1, %eax2901; AVX512F-NEXT: kmovw %eax, %k02902; AVX512F-NEXT: kmovw %edx, %k12903; AVX512F-NEXT: kshiftlw $15, %k1, %k12904; AVX512F-NEXT: kshiftrw $14, %k1, %k12905; AVX512F-NEXT: korw %k1, %k0, %k12906; AVX512F-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm02907; AVX512F-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}2908; AVX512F-NEXT: vmovdqa %xmm1, (%rdi)2909; AVX512F-NEXT: retq2910;2911; AVX512BW-LABEL: smulo_v2i64:2912; AVX512BW: # %bb.0:2913; AVX512BW-NEXT: vmovq %xmm1, %rax2914; AVX512BW-NEXT: vmovq %xmm0, %rcx2915; AVX512BW-NEXT: vpextrq $1, %xmm1, %rdx2916; AVX512BW-NEXT: vpextrq $1, %xmm0, %rsi2917; AVX512BW-NEXT: imulq %rdx, %rsi2918; AVX512BW-NEXT: seto %dl2919; AVX512BW-NEXT: vmovq %rsi, %xmm02920; AVX512BW-NEXT: imulq %rax, %rcx2921; AVX512BW-NEXT: vmovq %rcx, %xmm12922; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]2923; AVX512BW-NEXT: seto %al2924; AVX512BW-NEXT: andl $1, %eax2925; AVX512BW-NEXT: kmovw %eax, %k02926; AVX512BW-NEXT: kmovd %edx, %k12927; AVX512BW-NEXT: kshiftlw $15, %k1, %k12928; AVX512BW-NEXT: kshiftrw $14, %k1, %k12929; AVX512BW-NEXT: korw %k1, %k0, %k12930; AVX512BW-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm02931; AVX512BW-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}2932; AVX512BW-NEXT: vmovdqa %xmm1, (%rdi)2933; AVX512BW-NEXT: retq2934 %t = call {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)2935 %val = extractvalue {<2 x i64>, <2 x i1>} %t, 02936 %obit = extractvalue {<2 x i64>, <2 x i1>} %t, 12937 %res = sext <2 x i1> %obit to <2 x i32>2938 store <2 x i64> %val, ptr %p22939 ret <2 x i32> %res2940}2941 2942define <4 x i32> @smulo_v4i24(<4 x i24> %a0, <4 x i24> %a1, ptr %p2) nounwind {2943; SSE2-LABEL: smulo_v4i24:2944; SSE2: # %bb.0:2945; SSE2-NEXT: pslld $8, %xmm02946; SSE2-NEXT: psrad $8, %xmm02947; SSE2-NEXT: pslld $8, %xmm12948; SSE2-NEXT: psrad $8, %xmm12949; SSE2-NEXT: pxor %xmm3, %xmm32950; SSE2-NEXT: pxor %xmm2, %xmm22951; SSE2-NEXT: pcmpgtd %xmm1, %xmm22952; SSE2-NEXT: pand %xmm0, %xmm22953; SSE2-NEXT: pcmpgtd %xmm0, %xmm32954; SSE2-NEXT: pand %xmm1, %xmm32955; SSE2-NEXT: paddd %xmm2, %xmm32956; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]2957; SSE2-NEXT: pmuludq %xmm1, %xmm02958; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]2959; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]2960; SSE2-NEXT: pmuludq %xmm4, %xmm22961; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]2962; SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]2963; SSE2-NEXT: psubd %xmm3, %xmm52964; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]2965; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]2966; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]2967; SSE2-NEXT: movdqa %xmm3, %xmm12968; SSE2-NEXT: pslld $8, %xmm12969; SSE2-NEXT: psrad $8, %xmm12970; SSE2-NEXT: pcmpeqd %xmm3, %xmm12971; SSE2-NEXT: psrad $31, %xmm32972; SSE2-NEXT: pcmpeqd %xmm5, %xmm32973; SSE2-NEXT: pcmpeqd %xmm4, %xmm42974; SSE2-NEXT: pxor %xmm4, %xmm32975; SSE2-NEXT: pxor %xmm4, %xmm12976; SSE2-NEXT: por %xmm3, %xmm12977; SSE2-NEXT: movd %xmm0, %eax2978; SSE2-NEXT: movw %ax, (%rdi)2979; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]2980; SSE2-NEXT: movd %xmm0, %ecx2981; SSE2-NEXT: movw %cx, 6(%rdi)2982; SSE2-NEXT: movd %xmm2, %edx2983; SSE2-NEXT: movw %dx, 3(%rdi)2984; SSE2-NEXT: shrl $16, %eax2985; SSE2-NEXT: movb %al, 2(%rdi)2986; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]2987; SSE2-NEXT: movd %xmm0, %eax2988; SSE2-NEXT: movw %ax, 9(%rdi)2989; SSE2-NEXT: shrl $16, %ecx2990; SSE2-NEXT: movb %cl, 8(%rdi)2991; SSE2-NEXT: shrl $16, %edx2992; SSE2-NEXT: movb %dl, 5(%rdi)2993; SSE2-NEXT: shrl $16, %eax2994; SSE2-NEXT: movb %al, 11(%rdi)2995; SSE2-NEXT: movdqa %xmm1, %xmm02996; SSE2-NEXT: retq2997;2998; SSSE3-LABEL: smulo_v4i24:2999; SSSE3: # %bb.0:3000; SSSE3-NEXT: pslld $8, %xmm03001; SSSE3-NEXT: psrad $8, %xmm03002; SSSE3-NEXT: pslld $8, %xmm13003; SSSE3-NEXT: psrad $8, %xmm13004; SSSE3-NEXT: pxor %xmm3, %xmm33005; SSSE3-NEXT: pxor %xmm2, %xmm23006; SSSE3-NEXT: pcmpgtd %xmm1, %xmm23007; SSSE3-NEXT: pand %xmm0, %xmm23008; SSSE3-NEXT: pcmpgtd %xmm0, %xmm33009; SSSE3-NEXT: pand %xmm1, %xmm33010; SSSE3-NEXT: paddd %xmm2, %xmm33011; SSSE3-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]3012; SSSE3-NEXT: pmuludq %xmm1, %xmm03013; SSSE3-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]3014; SSSE3-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]3015; SSSE3-NEXT: pmuludq %xmm4, %xmm23016; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]3017; SSSE3-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]3018; SSSE3-NEXT: psubd %xmm3, %xmm53019; SSSE3-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]3020; SSSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]3021; SSSE3-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]3022; SSSE3-NEXT: movdqa %xmm3, %xmm13023; SSSE3-NEXT: pslld $8, %xmm13024; SSSE3-NEXT: psrad $8, %xmm13025; SSSE3-NEXT: pcmpeqd %xmm3, %xmm13026; SSSE3-NEXT: psrad $31, %xmm33027; SSSE3-NEXT: pcmpeqd %xmm5, %xmm33028; SSSE3-NEXT: pcmpeqd %xmm4, %xmm43029; SSSE3-NEXT: pxor %xmm4, %xmm33030; SSSE3-NEXT: pxor %xmm4, %xmm13031; SSSE3-NEXT: por %xmm3, %xmm13032; SSSE3-NEXT: movd %xmm0, %eax3033; SSSE3-NEXT: movw %ax, (%rdi)3034; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]3035; SSSE3-NEXT: movd %xmm0, %ecx3036; SSSE3-NEXT: movw %cx, 6(%rdi)3037; SSSE3-NEXT: movd %xmm2, %edx3038; SSSE3-NEXT: movw %dx, 3(%rdi)3039; SSSE3-NEXT: shrl $16, %eax3040; SSSE3-NEXT: movb %al, 2(%rdi)3041; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]3042; SSSE3-NEXT: movd %xmm0, %eax3043; SSSE3-NEXT: movw %ax, 9(%rdi)3044; SSSE3-NEXT: shrl $16, %ecx3045; SSSE3-NEXT: movb %cl, 8(%rdi)3046; SSSE3-NEXT: shrl $16, %edx3047; SSSE3-NEXT: movb %dl, 5(%rdi)3048; SSSE3-NEXT: shrl $16, %eax3049; SSSE3-NEXT: movb %al, 11(%rdi)3050; SSSE3-NEXT: movdqa %xmm1, %xmm03051; SSSE3-NEXT: retq3052;3053; SSE41-LABEL: smulo_v4i24:3054; SSE41: # %bb.0:3055; SSE41-NEXT: pslld $8, %xmm03056; SSE41-NEXT: psrad $8, %xmm03057; SSE41-NEXT: pslld $8, %xmm13058; SSE41-NEXT: psrad $8, %xmm13059; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]3060; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]3061; SSE41-NEXT: pmuldq %xmm2, %xmm33062; SSE41-NEXT: movdqa %xmm0, %xmm23063; SSE41-NEXT: pmuldq %xmm1, %xmm23064; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]3065; SSE41-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]3066; SSE41-NEXT: pmulld %xmm0, %xmm13067; SSE41-NEXT: movdqa %xmm1, %xmm03068; SSE41-NEXT: pslld $8, %xmm03069; SSE41-NEXT: psrad $8, %xmm03070; SSE41-NEXT: pcmpeqd %xmm1, %xmm03071; SSE41-NEXT: pextrd $3, %xmm1, %eax3072; SSE41-NEXT: pextrd $2, %xmm1, %ecx3073; SSE41-NEXT: pextrd $1, %xmm1, %edx3074; SSE41-NEXT: movd %xmm1, %esi3075; SSE41-NEXT: psrad $31, %xmm13076; SSE41-NEXT: pcmpeqd %xmm2, %xmm13077; SSE41-NEXT: pcmpeqd %xmm2, %xmm23078; SSE41-NEXT: pxor %xmm2, %xmm13079; SSE41-NEXT: pxor %xmm2, %xmm03080; SSE41-NEXT: por %xmm1, %xmm03081; SSE41-NEXT: movw %ax, 9(%rdi)3082; SSE41-NEXT: movw %cx, 6(%rdi)3083; SSE41-NEXT: movw %dx, 3(%rdi)3084; SSE41-NEXT: movw %si, (%rdi)3085; SSE41-NEXT: shrl $16, %eax3086; SSE41-NEXT: movb %al, 11(%rdi)3087; SSE41-NEXT: shrl $16, %ecx3088; SSE41-NEXT: movb %cl, 8(%rdi)3089; SSE41-NEXT: shrl $16, %edx3090; SSE41-NEXT: movb %dl, 5(%rdi)3091; SSE41-NEXT: shrl $16, %esi3092; SSE41-NEXT: movb %sil, 2(%rdi)3093; SSE41-NEXT: retq3094;3095; AVX1-LABEL: smulo_v4i24:3096; AVX1: # %bb.0:3097; AVX1-NEXT: vpslld $8, %xmm0, %xmm03098; AVX1-NEXT: vpsrad $8, %xmm0, %xmm03099; AVX1-NEXT: vpslld $8, %xmm1, %xmm13100; AVX1-NEXT: vpsrad $8, %xmm1, %xmm13101; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]3102; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]3103; AVX1-NEXT: vpmuldq %xmm2, %xmm3, %xmm23104; AVX1-NEXT: vpmuldq %xmm1, %xmm0, %xmm33105; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]3106; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]3107; AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm13108; AVX1-NEXT: vpsrad $31, %xmm1, %xmm03109; AVX1-NEXT: vpcmpeqd %xmm0, %xmm2, %xmm03110; AVX1-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm23111; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm03112; AVX1-NEXT: vpslld $8, %xmm1, %xmm33113; AVX1-NEXT: vpsrad $8, %xmm3, %xmm33114; AVX1-NEXT: vpcmpeqd %xmm1, %xmm3, %xmm33115; AVX1-NEXT: vpxor %xmm2, %xmm3, %xmm23116; AVX1-NEXT: vpor %xmm0, %xmm2, %xmm03117; AVX1-NEXT: vpextrd $3, %xmm1, %eax3118; AVX1-NEXT: movw %ax, 9(%rdi)3119; AVX1-NEXT: vpextrd $2, %xmm1, %ecx3120; AVX1-NEXT: movw %cx, 6(%rdi)3121; AVX1-NEXT: vpextrd $1, %xmm1, %edx3122; AVX1-NEXT: movw %dx, 3(%rdi)3123; AVX1-NEXT: vmovd %xmm1, %esi3124; AVX1-NEXT: movw %si, (%rdi)3125; AVX1-NEXT: shrl $16, %eax3126; AVX1-NEXT: movb %al, 11(%rdi)3127; AVX1-NEXT: shrl $16, %ecx3128; AVX1-NEXT: movb %cl, 8(%rdi)3129; AVX1-NEXT: shrl $16, %edx3130; AVX1-NEXT: movb %dl, 5(%rdi)3131; AVX1-NEXT: shrl $16, %esi3132; AVX1-NEXT: movb %sil, 2(%rdi)3133; AVX1-NEXT: retq3134;3135; AVX2-LABEL: smulo_v4i24:3136; AVX2: # %bb.0:3137; AVX2-NEXT: vpslld $8, %xmm0, %xmm03138; AVX2-NEXT: vpsrad $8, %xmm0, %xmm03139; AVX2-NEXT: vpslld $8, %xmm1, %xmm13140; AVX2-NEXT: vpsrad $8, %xmm1, %xmm13141; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]3142; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]3143; AVX2-NEXT: vpmuldq %xmm2, %xmm3, %xmm23144; AVX2-NEXT: vpmuldq %xmm1, %xmm0, %xmm33145; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]3146; AVX2-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]3147; AVX2-NEXT: vpmulld %xmm1, %xmm0, %xmm13148; AVX2-NEXT: vpsrad $31, %xmm1, %xmm03149; AVX2-NEXT: vpcmpeqd %xmm0, %xmm2, %xmm03150; AVX2-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm23151; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm03152; AVX2-NEXT: vpslld $8, %xmm1, %xmm33153; AVX2-NEXT: vpsrad $8, %xmm3, %xmm33154; AVX2-NEXT: vpcmpeqd %xmm1, %xmm3, %xmm33155; AVX2-NEXT: vpxor %xmm2, %xmm3, %xmm23156; AVX2-NEXT: vpor %xmm0, %xmm2, %xmm03157; AVX2-NEXT: vpextrd $3, %xmm1, %eax3158; AVX2-NEXT: movw %ax, 9(%rdi)3159; AVX2-NEXT: vpextrd $2, %xmm1, %ecx3160; AVX2-NEXT: movw %cx, 6(%rdi)3161; AVX2-NEXT: vpextrd $1, %xmm1, %edx3162; AVX2-NEXT: movw %dx, 3(%rdi)3163; AVX2-NEXT: vmovd %xmm1, %esi3164; AVX2-NEXT: movw %si, (%rdi)3165; AVX2-NEXT: shrl $16, %eax3166; AVX2-NEXT: movb %al, 11(%rdi)3167; AVX2-NEXT: shrl $16, %ecx3168; AVX2-NEXT: movb %cl, 8(%rdi)3169; AVX2-NEXT: shrl $16, %edx3170; AVX2-NEXT: movb %dl, 5(%rdi)3171; AVX2-NEXT: shrl $16, %esi3172; AVX2-NEXT: movb %sil, 2(%rdi)3173; AVX2-NEXT: retq3174;3175; AVX512-LABEL: smulo_v4i24:3176; AVX512: # %bb.0:3177; AVX512-NEXT: vpslld $8, %xmm1, %xmm13178; AVX512-NEXT: vpsrad $8, %xmm1, %xmm13179; AVX512-NEXT: vpslld $8, %xmm0, %xmm03180; AVX512-NEXT: vpsrad $8, %xmm0, %xmm03181; AVX512-NEXT: vpmuldq %xmm1, %xmm0, %xmm23182; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]3183; AVX512-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]3184; AVX512-NEXT: vpmuldq %xmm3, %xmm4, %xmm33185; AVX512-NEXT: vpmovsxbd {{.*#+}} xmm4 = [1,5,3,7]3186; AVX512-NEXT: vpermi2d %xmm3, %xmm2, %xmm43187; AVX512-NEXT: vpmulld %xmm1, %xmm0, %xmm13188; AVX512-NEXT: vpsrad $31, %xmm1, %xmm03189; AVX512-NEXT: vpcmpneqd %xmm0, %xmm4, %k03190; AVX512-NEXT: vpslld $8, %xmm1, %xmm03191; AVX512-NEXT: vpsrad $8, %xmm0, %xmm03192; AVX512-NEXT: vpcmpneqd %xmm1, %xmm0, %k13193; AVX512-NEXT: korw %k0, %k1, %k13194; AVX512-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm03195; AVX512-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}3196; AVX512-NEXT: vpextrd $3, %xmm1, %eax3197; AVX512-NEXT: movw %ax, 9(%rdi)3198; AVX512-NEXT: vpextrd $2, %xmm1, %ecx3199; AVX512-NEXT: movw %cx, 6(%rdi)3200; AVX512-NEXT: vpextrd $1, %xmm1, %edx3201; AVX512-NEXT: movw %dx, 3(%rdi)3202; AVX512-NEXT: vmovd %xmm1, %esi3203; AVX512-NEXT: movw %si, (%rdi)3204; AVX512-NEXT: shrl $16, %eax3205; AVX512-NEXT: movb %al, 11(%rdi)3206; AVX512-NEXT: shrl $16, %ecx3207; AVX512-NEXT: movb %cl, 8(%rdi)3208; AVX512-NEXT: shrl $16, %edx3209; AVX512-NEXT: movb %dl, 5(%rdi)3210; AVX512-NEXT: shrl $16, %esi3211; AVX512-NEXT: movb %sil, 2(%rdi)3212; AVX512-NEXT: retq3213 %t = call {<4 x i24>, <4 x i1>} @llvm.smul.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)3214 %val = extractvalue {<4 x i24>, <4 x i1>} %t, 03215 %obit = extractvalue {<4 x i24>, <4 x i1>} %t, 13216 %res = sext <4 x i1> %obit to <4 x i32>3217 store <4 x i24> %val, ptr %p23218 ret <4 x i32> %res3219}3220 3221define <4 x i32> @smulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, ptr %p2) nounwind {3222; SSE2-LABEL: smulo_v4i1:3223; SSE2: # %bb.0:3224; SSE2-NEXT: pand %xmm1, %xmm03225; SSE2-NEXT: pslld $31, %xmm03226; SSE2-NEXT: movmskps %xmm0, %eax3227; SSE2-NEXT: psrad $31, %xmm03228; SSE2-NEXT: movb %al, (%rdi)3229; SSE2-NEXT: retq3230;3231; SSSE3-LABEL: smulo_v4i1:3232; SSSE3: # %bb.0:3233; SSSE3-NEXT: pand %xmm1, %xmm03234; SSSE3-NEXT: pslld $31, %xmm03235; SSSE3-NEXT: movmskps %xmm0, %eax3236; SSSE3-NEXT: psrad $31, %xmm03237; SSSE3-NEXT: movb %al, (%rdi)3238; SSSE3-NEXT: retq3239;3240; SSE41-LABEL: smulo_v4i1:3241; SSE41: # %bb.0:3242; SSE41-NEXT: pand %xmm1, %xmm03243; SSE41-NEXT: pslld $31, %xmm03244; SSE41-NEXT: movmskps %xmm0, %eax3245; SSE41-NEXT: psrad $31, %xmm03246; SSE41-NEXT: movb %al, (%rdi)3247; SSE41-NEXT: retq3248;3249; AVX-LABEL: smulo_v4i1:3250; AVX: # %bb.0:3251; AVX-NEXT: vpand %xmm1, %xmm0, %xmm03252; AVX-NEXT: vpslld $31, %xmm0, %xmm13253; AVX-NEXT: vpsrad $31, %xmm1, %xmm03254; AVX-NEXT: vmovmskps %xmm1, %eax3255; AVX-NEXT: movb %al, (%rdi)3256; AVX-NEXT: retq3257;3258; AVX512F-LABEL: smulo_v4i1:3259; AVX512F: # %bb.0:3260; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm03261; AVX512F-NEXT: vpslld $31, %xmm0, %xmm03262; AVX512F-NEXT: vptestmd %xmm0, %xmm0, %k03263; AVX512F-NEXT: vpsrad $31, %xmm0, %xmm03264; AVX512F-NEXT: kmovw %k0, %eax3265; AVX512F-NEXT: movb %al, (%rdi)3266; AVX512F-NEXT: retq3267;3268; AVX512BW-LABEL: smulo_v4i1:3269; AVX512BW: # %bb.0:3270; AVX512BW-NEXT: vpand %xmm1, %xmm0, %xmm03271; AVX512BW-NEXT: vpslld $31, %xmm0, %xmm03272; AVX512BW-NEXT: vptestmd %xmm0, %xmm0, %k03273; AVX512BW-NEXT: vpsrad $31, %xmm0, %xmm03274; AVX512BW-NEXT: kmovd %k0, %eax3275; AVX512BW-NEXT: movb %al, (%rdi)3276; AVX512BW-NEXT: retq3277 %t = call {<4 x i1>, <4 x i1>} @llvm.smul.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)3278 %val = extractvalue {<4 x i1>, <4 x i1>} %t, 03279 %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 13280 %res = sext <4 x i1> %obit to <4 x i32>3281 store <4 x i1> %val, ptr %p23282 ret <4 x i32> %res3283}3284 3285define <2 x i32> @smulo_v2i128(<2 x i128> %a0, <2 x i128> %a1, ptr %p2) nounwind {3286; SSE2-LABEL: smulo_v2i128:3287; SSE2: # %bb.0:3288; SSE2-NEXT: pushq %rbp3289; SSE2-NEXT: pushq %r153290; SSE2-NEXT: pushq %r143291; SSE2-NEXT: pushq %r133292; SSE2-NEXT: pushq %r123293; SSE2-NEXT: pushq %rbx3294; SSE2-NEXT: movq %r9, %r103295; SSE2-NEXT: movq %rcx, %rbx3296; SSE2-NEXT: movq %rdx, %rcx3297; SSE2-NEXT: movq %rsi, %r113298; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r153299; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r123300; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %r133301; SSE2-NEXT: movq %rsi, %rbp3302; SSE2-NEXT: sarq $63, %rbp3303; SSE2-NEXT: imulq %r8, %rbp3304; SSE2-NEXT: movq %rdi, %rax3305; SSE2-NEXT: mulq %r83306; SSE2-NEXT: movq %rdx, %rsi3307; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3308; SSE2-NEXT: movq %r11, %rax3309; SSE2-NEXT: mulq %r83310; SSE2-NEXT: movq %rdx, %r83311; SSE2-NEXT: movq %rax, %r143312; SSE2-NEXT: addq %rsi, %r143313; SSE2-NEXT: adcq %rbp, %r83314; SSE2-NEXT: movq %r8, %rbp3315; SSE2-NEXT: sarq $63, %rbp3316; SSE2-NEXT: sarq $63, %r93317; SSE2-NEXT: imulq %rdi, %r93318; SSE2-NEXT: movq %rdi, %rax3319; SSE2-NEXT: mulq %r103320; SSE2-NEXT: movq %rdx, %rdi3321; SSE2-NEXT: movq %rax, %rsi3322; SSE2-NEXT: addq %r14, %rsi3323; SSE2-NEXT: adcq %r9, %rdi3324; SSE2-NEXT: movq %rdi, %r93325; SSE2-NEXT: sarq $63, %r93326; SSE2-NEXT: addq %r8, %rdi3327; SSE2-NEXT: adcq %rbp, %r93328; SSE2-NEXT: movq %r11, %rax3329; SSE2-NEXT: imulq %r103330; SSE2-NEXT: addq %rdi, %rax3331; SSE2-NEXT: adcq %r9, %rdx3332; SSE2-NEXT: movq %rsi, 8(%r15)3333; SSE2-NEXT: sarq $63, %rsi3334; SSE2-NEXT: xorq %rsi, %rdx3335; SSE2-NEXT: xorq %rax, %rsi3336; SSE2-NEXT: xorl %r11d, %r11d3337; SSE2-NEXT: orq %rdx, %rsi3338; SSE2-NEXT: setne %r11b3339; SSE2-NEXT: movq %rbx, %r103340; SSE2-NEXT: sarq $63, %r103341; SSE2-NEXT: imulq %r13, %r103342; SSE2-NEXT: movq %rcx, %rax3343; SSE2-NEXT: mulq %r133344; SSE2-NEXT: movq %rdx, %rdi3345; SSE2-NEXT: movq %rax, %rsi3346; SSE2-NEXT: movq %rbx, %rax3347; SSE2-NEXT: mulq %r133348; SSE2-NEXT: movq %rdx, %r83349; SSE2-NEXT: movq %rax, %r93350; SSE2-NEXT: addq %rdi, %r93351; SSE2-NEXT: adcq %r10, %r83352; SSE2-NEXT: movq %r8, %r143353; SSE2-NEXT: sarq $63, %r143354; SSE2-NEXT: movq %r12, %r133355; SSE2-NEXT: sarq $63, %r133356; SSE2-NEXT: imulq %rcx, %r133357; SSE2-NEXT: movq %rcx, %rax3358; SSE2-NEXT: mulq %r123359; SSE2-NEXT: movq %rdx, %rdi3360; SSE2-NEXT: movq %rax, %r103361; SSE2-NEXT: addq %r9, %r103362; SSE2-NEXT: adcq %r13, %rdi3363; SSE2-NEXT: movq %rdi, %rcx3364; SSE2-NEXT: sarq $63, %rcx3365; SSE2-NEXT: addq %r8, %rdi3366; SSE2-NEXT: adcq %r14, %rcx3367; SSE2-NEXT: movq %rbx, %rax3368; SSE2-NEXT: imulq %r123369; SSE2-NEXT: addq %rdi, %rax3370; SSE2-NEXT: adcq %rcx, %rdx3371; SSE2-NEXT: movq %r10, 24(%r15)3372; SSE2-NEXT: sarq $63, %r103373; SSE2-NEXT: xorq %r10, %rdx3374; SSE2-NEXT: xorq %rax, %r103375; SSE2-NEXT: xorl %eax, %eax3376; SSE2-NEXT: orq %rdx, %r103377; SSE2-NEXT: setne %al3378; SSE2-NEXT: negl %eax3379; SSE2-NEXT: movd %eax, %xmm13380; SSE2-NEXT: negl %r11d3381; SSE2-NEXT: movd %r11d, %xmm03382; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]3383; SSE2-NEXT: movq %rsi, 16(%r15)3384; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload3385; SSE2-NEXT: movq %rax, (%r15)3386; SSE2-NEXT: popq %rbx3387; SSE2-NEXT: popq %r123388; SSE2-NEXT: popq %r133389; SSE2-NEXT: popq %r143390; SSE2-NEXT: popq %r153391; SSE2-NEXT: popq %rbp3392; SSE2-NEXT: retq3393;3394; SSSE3-LABEL: smulo_v2i128:3395; SSSE3: # %bb.0:3396; SSSE3-NEXT: pushq %rbp3397; SSSE3-NEXT: pushq %r153398; SSSE3-NEXT: pushq %r143399; SSSE3-NEXT: pushq %r133400; SSSE3-NEXT: pushq %r123401; SSSE3-NEXT: pushq %rbx3402; SSSE3-NEXT: movq %r9, %r103403; SSSE3-NEXT: movq %rcx, %rbx3404; SSSE3-NEXT: movq %rdx, %rcx3405; SSSE3-NEXT: movq %rsi, %r113406; SSSE3-NEXT: movq {{[0-9]+}}(%rsp), %r153407; SSSE3-NEXT: movq {{[0-9]+}}(%rsp), %r123408; SSSE3-NEXT: movq {{[0-9]+}}(%rsp), %r133409; SSSE3-NEXT: movq %rsi, %rbp3410; SSSE3-NEXT: sarq $63, %rbp3411; SSSE3-NEXT: imulq %r8, %rbp3412; SSSE3-NEXT: movq %rdi, %rax3413; SSSE3-NEXT: mulq %r83414; SSSE3-NEXT: movq %rdx, %rsi3415; SSSE3-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3416; SSSE3-NEXT: movq %r11, %rax3417; SSSE3-NEXT: mulq %r83418; SSSE3-NEXT: movq %rdx, %r83419; SSSE3-NEXT: movq %rax, %r143420; SSSE3-NEXT: addq %rsi, %r143421; SSSE3-NEXT: adcq %rbp, %r83422; SSSE3-NEXT: movq %r8, %rbp3423; SSSE3-NEXT: sarq $63, %rbp3424; SSSE3-NEXT: sarq $63, %r93425; SSSE3-NEXT: imulq %rdi, %r93426; SSSE3-NEXT: movq %rdi, %rax3427; SSSE3-NEXT: mulq %r103428; SSSE3-NEXT: movq %rdx, %rdi3429; SSSE3-NEXT: movq %rax, %rsi3430; SSSE3-NEXT: addq %r14, %rsi3431; SSSE3-NEXT: adcq %r9, %rdi3432; SSSE3-NEXT: movq %rdi, %r93433; SSSE3-NEXT: sarq $63, %r93434; SSSE3-NEXT: addq %r8, %rdi3435; SSSE3-NEXT: adcq %rbp, %r93436; SSSE3-NEXT: movq %r11, %rax3437; SSSE3-NEXT: imulq %r103438; SSSE3-NEXT: addq %rdi, %rax3439; SSSE3-NEXT: adcq %r9, %rdx3440; SSSE3-NEXT: movq %rsi, 8(%r15)3441; SSSE3-NEXT: sarq $63, %rsi3442; SSSE3-NEXT: xorq %rsi, %rdx3443; SSSE3-NEXT: xorq %rax, %rsi3444; SSSE3-NEXT: xorl %r11d, %r11d3445; SSSE3-NEXT: orq %rdx, %rsi3446; SSSE3-NEXT: setne %r11b3447; SSSE3-NEXT: movq %rbx, %r103448; SSSE3-NEXT: sarq $63, %r103449; SSSE3-NEXT: imulq %r13, %r103450; SSSE3-NEXT: movq %rcx, %rax3451; SSSE3-NEXT: mulq %r133452; SSSE3-NEXT: movq %rdx, %rdi3453; SSSE3-NEXT: movq %rax, %rsi3454; SSSE3-NEXT: movq %rbx, %rax3455; SSSE3-NEXT: mulq %r133456; SSSE3-NEXT: movq %rdx, %r83457; SSSE3-NEXT: movq %rax, %r93458; SSSE3-NEXT: addq %rdi, %r93459; SSSE3-NEXT: adcq %r10, %r83460; SSSE3-NEXT: movq %r8, %r143461; SSSE3-NEXT: sarq $63, %r143462; SSSE3-NEXT: movq %r12, %r133463; SSSE3-NEXT: sarq $63, %r133464; SSSE3-NEXT: imulq %rcx, %r133465; SSSE3-NEXT: movq %rcx, %rax3466; SSSE3-NEXT: mulq %r123467; SSSE3-NEXT: movq %rdx, %rdi3468; SSSE3-NEXT: movq %rax, %r103469; SSSE3-NEXT: addq %r9, %r103470; SSSE3-NEXT: adcq %r13, %rdi3471; SSSE3-NEXT: movq %rdi, %rcx3472; SSSE3-NEXT: sarq $63, %rcx3473; SSSE3-NEXT: addq %r8, %rdi3474; SSSE3-NEXT: adcq %r14, %rcx3475; SSSE3-NEXT: movq %rbx, %rax3476; SSSE3-NEXT: imulq %r123477; SSSE3-NEXT: addq %rdi, %rax3478; SSSE3-NEXT: adcq %rcx, %rdx3479; SSSE3-NEXT: movq %r10, 24(%r15)3480; SSSE3-NEXT: sarq $63, %r103481; SSSE3-NEXT: xorq %r10, %rdx3482; SSSE3-NEXT: xorq %rax, %r103483; SSSE3-NEXT: xorl %eax, %eax3484; SSSE3-NEXT: orq %rdx, %r103485; SSSE3-NEXT: setne %al3486; SSSE3-NEXT: negl %eax3487; SSSE3-NEXT: movd %eax, %xmm13488; SSSE3-NEXT: negl %r11d3489; SSSE3-NEXT: movd %r11d, %xmm03490; SSSE3-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]3491; SSSE3-NEXT: movq %rsi, 16(%r15)3492; SSSE3-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload3493; SSSE3-NEXT: movq %rax, (%r15)3494; SSSE3-NEXT: popq %rbx3495; SSSE3-NEXT: popq %r123496; SSSE3-NEXT: popq %r133497; SSSE3-NEXT: popq %r143498; SSSE3-NEXT: popq %r153499; SSSE3-NEXT: popq %rbp3500; SSSE3-NEXT: retq3501;3502; SSE41-LABEL: smulo_v2i128:3503; SSE41: # %bb.0:3504; SSE41-NEXT: pushq %rbp3505; SSE41-NEXT: pushq %r153506; SSE41-NEXT: pushq %r143507; SSE41-NEXT: pushq %r133508; SSE41-NEXT: pushq %r123509; SSE41-NEXT: pushq %rbx3510; SSE41-NEXT: movq %r9, %r103511; SSE41-NEXT: movq %rcx, %rbx3512; SSE41-NEXT: movq %rdx, %rcx3513; SSE41-NEXT: movq %rsi, %r113514; SSE41-NEXT: movq {{[0-9]+}}(%rsp), %r153515; SSE41-NEXT: movq {{[0-9]+}}(%rsp), %r123516; SSE41-NEXT: movq {{[0-9]+}}(%rsp), %r133517; SSE41-NEXT: movq %rsi, %rbp3518; SSE41-NEXT: sarq $63, %rbp3519; SSE41-NEXT: imulq %r8, %rbp3520; SSE41-NEXT: movq %rdi, %rax3521; SSE41-NEXT: mulq %r83522; SSE41-NEXT: movq %rdx, %rsi3523; SSE41-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3524; SSE41-NEXT: movq %r11, %rax3525; SSE41-NEXT: mulq %r83526; SSE41-NEXT: movq %rdx, %r83527; SSE41-NEXT: movq %rax, %r143528; SSE41-NEXT: addq %rsi, %r143529; SSE41-NEXT: adcq %rbp, %r83530; SSE41-NEXT: movq %r8, %rbp3531; SSE41-NEXT: sarq $63, %rbp3532; SSE41-NEXT: sarq $63, %r93533; SSE41-NEXT: imulq %rdi, %r93534; SSE41-NEXT: movq %rdi, %rax3535; SSE41-NEXT: mulq %r103536; SSE41-NEXT: movq %rdx, %rdi3537; SSE41-NEXT: movq %rax, %rsi3538; SSE41-NEXT: addq %r14, %rsi3539; SSE41-NEXT: adcq %r9, %rdi3540; SSE41-NEXT: movq %rdi, %r93541; SSE41-NEXT: sarq $63, %r93542; SSE41-NEXT: addq %r8, %rdi3543; SSE41-NEXT: adcq %rbp, %r93544; SSE41-NEXT: movq %r11, %rax3545; SSE41-NEXT: imulq %r103546; SSE41-NEXT: addq %rdi, %rax3547; SSE41-NEXT: adcq %r9, %rdx3548; SSE41-NEXT: movq %rsi, 8(%r15)3549; SSE41-NEXT: sarq $63, %rsi3550; SSE41-NEXT: xorq %rsi, %rdx3551; SSE41-NEXT: xorq %rax, %rsi3552; SSE41-NEXT: xorl %r11d, %r11d3553; SSE41-NEXT: orq %rdx, %rsi3554; SSE41-NEXT: setne %r11b3555; SSE41-NEXT: movq %rbx, %r103556; SSE41-NEXT: sarq $63, %r103557; SSE41-NEXT: imulq %r13, %r103558; SSE41-NEXT: movq %rcx, %rax3559; SSE41-NEXT: mulq %r133560; SSE41-NEXT: movq %rdx, %rdi3561; SSE41-NEXT: movq %rax, %rsi3562; SSE41-NEXT: movq %rbx, %rax3563; SSE41-NEXT: mulq %r133564; SSE41-NEXT: movq %rdx, %r83565; SSE41-NEXT: movq %rax, %r93566; SSE41-NEXT: addq %rdi, %r93567; SSE41-NEXT: adcq %r10, %r83568; SSE41-NEXT: movq %r8, %r143569; SSE41-NEXT: sarq $63, %r143570; SSE41-NEXT: movq %r12, %r133571; SSE41-NEXT: sarq $63, %r133572; SSE41-NEXT: imulq %rcx, %r133573; SSE41-NEXT: movq %rcx, %rax3574; SSE41-NEXT: mulq %r123575; SSE41-NEXT: movq %rdx, %rdi3576; SSE41-NEXT: movq %rax, %r103577; SSE41-NEXT: addq %r9, %r103578; SSE41-NEXT: adcq %r13, %rdi3579; SSE41-NEXT: movq %rdi, %rcx3580; SSE41-NEXT: sarq $63, %rcx3581; SSE41-NEXT: addq %r8, %rdi3582; SSE41-NEXT: adcq %r14, %rcx3583; SSE41-NEXT: movq %rbx, %rax3584; SSE41-NEXT: imulq %r123585; SSE41-NEXT: addq %rdi, %rax3586; SSE41-NEXT: adcq %rcx, %rdx3587; SSE41-NEXT: movq %r10, 24(%r15)3588; SSE41-NEXT: sarq $63, %r103589; SSE41-NEXT: xorq %r10, %rdx3590; SSE41-NEXT: xorq %rax, %r103591; SSE41-NEXT: xorl %eax, %eax3592; SSE41-NEXT: orq %rdx, %r103593; SSE41-NEXT: setne %al3594; SSE41-NEXT: negl %eax3595; SSE41-NEXT: negl %r11d3596; SSE41-NEXT: movd %r11d, %xmm03597; SSE41-NEXT: pinsrd $1, %eax, %xmm03598; SSE41-NEXT: movq %rsi, 16(%r15)3599; SSE41-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload3600; SSE41-NEXT: movq %rax, (%r15)3601; SSE41-NEXT: popq %rbx3602; SSE41-NEXT: popq %r123603; SSE41-NEXT: popq %r133604; SSE41-NEXT: popq %r143605; SSE41-NEXT: popq %r153606; SSE41-NEXT: popq %rbp3607; SSE41-NEXT: retq3608;3609; AVX-LABEL: smulo_v2i128:3610; AVX: # %bb.0:3611; AVX-NEXT: pushq %rbp3612; AVX-NEXT: pushq %r153613; AVX-NEXT: pushq %r143614; AVX-NEXT: pushq %r133615; AVX-NEXT: pushq %r123616; AVX-NEXT: pushq %rbx3617; AVX-NEXT: movq %r9, %r103618; AVX-NEXT: movq %rcx, %rbx3619; AVX-NEXT: movq %rdx, %rcx3620; AVX-NEXT: movq %rsi, %r113621; AVX-NEXT: movq {{[0-9]+}}(%rsp), %r153622; AVX-NEXT: movq {{[0-9]+}}(%rsp), %r133623; AVX-NEXT: movq {{[0-9]+}}(%rsp), %r123624; AVX-NEXT: movq %rsi, %rbp3625; AVX-NEXT: sarq $63, %rbp3626; AVX-NEXT: imulq %r8, %rbp3627; AVX-NEXT: movq %rdi, %rax3628; AVX-NEXT: mulq %r83629; AVX-NEXT: movq %rdx, %rsi3630; AVX-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3631; AVX-NEXT: movq %r11, %rax3632; AVX-NEXT: mulq %r83633; AVX-NEXT: movq %rdx, %r83634; AVX-NEXT: movq %rax, %r143635; AVX-NEXT: addq %rsi, %r143636; AVX-NEXT: adcq %rbp, %r83637; AVX-NEXT: movq %r8, %rbp3638; AVX-NEXT: sarq $63, %rbp3639; AVX-NEXT: sarq $63, %r93640; AVX-NEXT: imulq %rdi, %r93641; AVX-NEXT: movq %rdi, %rax3642; AVX-NEXT: mulq %r103643; AVX-NEXT: movq %rdx, %rdi3644; AVX-NEXT: movq %rax, %rsi3645; AVX-NEXT: addq %r14, %rsi3646; AVX-NEXT: adcq %r9, %rdi3647; AVX-NEXT: movq %rdi, %r93648; AVX-NEXT: sarq $63, %r93649; AVX-NEXT: addq %r8, %rdi3650; AVX-NEXT: adcq %rbp, %r93651; AVX-NEXT: movq %r11, %rax3652; AVX-NEXT: imulq %r103653; AVX-NEXT: addq %rdi, %rax3654; AVX-NEXT: adcq %r9, %rdx3655; AVX-NEXT: movq %rsi, 8(%r15)3656; AVX-NEXT: sarq $63, %rsi3657; AVX-NEXT: xorq %rsi, %rdx3658; AVX-NEXT: xorq %rax, %rsi3659; AVX-NEXT: xorl %r11d, %r11d3660; AVX-NEXT: orq %rdx, %rsi3661; AVX-NEXT: setne %r11b3662; AVX-NEXT: movq %rbx, %r103663; AVX-NEXT: sarq $63, %r103664; AVX-NEXT: imulq %r13, %r103665; AVX-NEXT: movq %rcx, %rax3666; AVX-NEXT: mulq %r133667; AVX-NEXT: movq %rdx, %rdi3668; AVX-NEXT: movq %rax, %rsi3669; AVX-NEXT: movq %rbx, %rax3670; AVX-NEXT: mulq %r133671; AVX-NEXT: movq %rdx, %r83672; AVX-NEXT: movq %rax, %r93673; AVX-NEXT: addq %rdi, %r93674; AVX-NEXT: adcq %r10, %r83675; AVX-NEXT: movq %r8, %r143676; AVX-NEXT: sarq $63, %r143677; AVX-NEXT: movq %r12, %r133678; AVX-NEXT: sarq $63, %r133679; AVX-NEXT: imulq %rcx, %r133680; AVX-NEXT: movq %rcx, %rax3681; AVX-NEXT: mulq %r123682; AVX-NEXT: movq %rdx, %rdi3683; AVX-NEXT: movq %rax, %r103684; AVX-NEXT: addq %r9, %r103685; AVX-NEXT: adcq %r13, %rdi3686; AVX-NEXT: movq %rdi, %rcx3687; AVX-NEXT: sarq $63, %rcx3688; AVX-NEXT: addq %r8, %rdi3689; AVX-NEXT: adcq %r14, %rcx3690; AVX-NEXT: movq %rbx, %rax3691; AVX-NEXT: imulq %r123692; AVX-NEXT: addq %rdi, %rax3693; AVX-NEXT: adcq %rcx, %rdx3694; AVX-NEXT: movq %r10, 24(%r15)3695; AVX-NEXT: sarq $63, %r103696; AVX-NEXT: xorq %r10, %rdx3697; AVX-NEXT: xorq %rax, %r103698; AVX-NEXT: xorl %eax, %eax3699; AVX-NEXT: orq %rdx, %r103700; AVX-NEXT: setne %al3701; AVX-NEXT: negl %eax3702; AVX-NEXT: negl %r11d3703; AVX-NEXT: vmovd %r11d, %xmm03704; AVX-NEXT: vpinsrd $1, %eax, %xmm0, %xmm03705; AVX-NEXT: movq %rsi, 16(%r15)3706; AVX-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload3707; AVX-NEXT: movq %rax, (%r15)3708; AVX-NEXT: popq %rbx3709; AVX-NEXT: popq %r123710; AVX-NEXT: popq %r133711; AVX-NEXT: popq %r143712; AVX-NEXT: popq %r153713; AVX-NEXT: popq %rbp3714; AVX-NEXT: retq3715;3716; AVX512F-LABEL: smulo_v2i128:3717; AVX512F: # %bb.0:3718; AVX512F-NEXT: pushq %rbp3719; AVX512F-NEXT: pushq %r153720; AVX512F-NEXT: pushq %r143721; AVX512F-NEXT: pushq %r133722; AVX512F-NEXT: pushq %r123723; AVX512F-NEXT: pushq %rbx3724; AVX512F-NEXT: movq %rcx, %r113725; AVX512F-NEXT: movq %rdx, %r103726; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r123727; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r143728; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %rcx3729; AVX512F-NEXT: movq %r11, %rbp3730; AVX512F-NEXT: sarq $63, %rbp3731; AVX512F-NEXT: imulq %r14, %rbp3732; AVX512F-NEXT: movq %rdx, %rax3733; AVX512F-NEXT: mulq %r143734; AVX512F-NEXT: movq %rdx, %rbx3735; AVX512F-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3736; AVX512F-NEXT: movq %r11, %rax3737; AVX512F-NEXT: mulq %r143738; AVX512F-NEXT: movq %rdx, %r143739; AVX512F-NEXT: movq %rax, %r153740; AVX512F-NEXT: addq %rbx, %r153741; AVX512F-NEXT: adcq %rbp, %r143742; AVX512F-NEXT: movq %r14, %rbp3743; AVX512F-NEXT: sarq $63, %rbp3744; AVX512F-NEXT: movq %rcx, %r133745; AVX512F-NEXT: sarq $63, %r133746; AVX512F-NEXT: imulq %r10, %r133747; AVX512F-NEXT: movq %r10, %rax3748; AVX512F-NEXT: mulq %rcx3749; AVX512F-NEXT: movq %rdx, %rbx3750; AVX512F-NEXT: movq %rax, %r103751; AVX512F-NEXT: addq %r15, %r103752; AVX512F-NEXT: adcq %r13, %rbx3753; AVX512F-NEXT: movq %rbx, %r153754; AVX512F-NEXT: sarq $63, %r153755; AVX512F-NEXT: addq %r14, %rbx3756; AVX512F-NEXT: adcq %rbp, %r153757; AVX512F-NEXT: movq %r11, %rax3758; AVX512F-NEXT: imulq %rcx3759; AVX512F-NEXT: addq %rbx, %rax3760; AVX512F-NEXT: adcq %r15, %rdx3761; AVX512F-NEXT: movq %r10, 24(%r12)3762; AVX512F-NEXT: sarq $63, %r103763; AVX512F-NEXT: xorq %r10, %rdx3764; AVX512F-NEXT: xorq %rax, %r103765; AVX512F-NEXT: orq %rdx, %r103766; AVX512F-NEXT: setne %al3767; AVX512F-NEXT: kmovw %eax, %k03768; AVX512F-NEXT: movq %rsi, %rcx3769; AVX512F-NEXT: sarq $63, %rcx3770; AVX512F-NEXT: imulq %r8, %rcx3771; AVX512F-NEXT: movq %rdi, %rax3772; AVX512F-NEXT: mulq %r83773; AVX512F-NEXT: movq %rdx, %r113774; AVX512F-NEXT: movq %rax, %r103775; AVX512F-NEXT: movq %rsi, %rax3776; AVX512F-NEXT: mulq %r83777; AVX512F-NEXT: movq %rdx, %r83778; AVX512F-NEXT: movq %rax, %rbx3779; AVX512F-NEXT: addq %r11, %rbx3780; AVX512F-NEXT: adcq %rcx, %r83781; AVX512F-NEXT: movq %r8, %rcx3782; AVX512F-NEXT: sarq $63, %rcx3783; AVX512F-NEXT: movq %r9, %r143784; AVX512F-NEXT: sarq $63, %r143785; AVX512F-NEXT: imulq %rdi, %r143786; AVX512F-NEXT: movq %rdi, %rax3787; AVX512F-NEXT: mulq %r93788; AVX512F-NEXT: movq %rdx, %rdi3789; AVX512F-NEXT: movq %rax, %r113790; AVX512F-NEXT: addq %rbx, %r113791; AVX512F-NEXT: adcq %r14, %rdi3792; AVX512F-NEXT: movq %rdi, %rbx3793; AVX512F-NEXT: sarq $63, %rbx3794; AVX512F-NEXT: addq %r8, %rdi3795; AVX512F-NEXT: adcq %rcx, %rbx3796; AVX512F-NEXT: movq %rsi, %rax3797; AVX512F-NEXT: imulq %r93798; AVX512F-NEXT: addq %rdi, %rax3799; AVX512F-NEXT: adcq %rbx, %rdx3800; AVX512F-NEXT: movq %r11, 8(%r12)3801; AVX512F-NEXT: sarq $63, %r113802; AVX512F-NEXT: xorq %r11, %rdx3803; AVX512F-NEXT: xorq %rax, %r113804; AVX512F-NEXT: orq %rdx, %r113805; AVX512F-NEXT: setne %al3806; AVX512F-NEXT: andl $1, %eax3807; AVX512F-NEXT: kmovw %eax, %k13808; AVX512F-NEXT: kshiftlw $1, %k0, %k03809; AVX512F-NEXT: korw %k0, %k1, %k13810; AVX512F-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm03811; AVX512F-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}3812; AVX512F-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload3813; AVX512F-NEXT: movq %rax, 16(%r12)3814; AVX512F-NEXT: movq %r10, (%r12)3815; AVX512F-NEXT: popq %rbx3816; AVX512F-NEXT: popq %r123817; AVX512F-NEXT: popq %r133818; AVX512F-NEXT: popq %r143819; AVX512F-NEXT: popq %r153820; AVX512F-NEXT: popq %rbp3821; AVX512F-NEXT: retq3822;3823; AVX512BW-LABEL: smulo_v2i128:3824; AVX512BW: # %bb.0:3825; AVX512BW-NEXT: pushq %rbp3826; AVX512BW-NEXT: pushq %r153827; AVX512BW-NEXT: pushq %r143828; AVX512BW-NEXT: pushq %r133829; AVX512BW-NEXT: pushq %r123830; AVX512BW-NEXT: pushq %rbx3831; AVX512BW-NEXT: movq %rcx, %r113832; AVX512BW-NEXT: movq %rdx, %r103833; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %r123834; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %r143835; AVX512BW-NEXT: movq {{[0-9]+}}(%rsp), %rcx3836; AVX512BW-NEXT: movq %r11, %rbp3837; AVX512BW-NEXT: sarq $63, %rbp3838; AVX512BW-NEXT: imulq %r14, %rbp3839; AVX512BW-NEXT: movq %rdx, %rax3840; AVX512BW-NEXT: mulq %r143841; AVX512BW-NEXT: movq %rdx, %rbx3842; AVX512BW-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3843; AVX512BW-NEXT: movq %r11, %rax3844; AVX512BW-NEXT: mulq %r143845; AVX512BW-NEXT: movq %rdx, %r143846; AVX512BW-NEXT: movq %rax, %r153847; AVX512BW-NEXT: addq %rbx, %r153848; AVX512BW-NEXT: adcq %rbp, %r143849; AVX512BW-NEXT: movq %r14, %rbp3850; AVX512BW-NEXT: sarq $63, %rbp3851; AVX512BW-NEXT: movq %rcx, %r133852; AVX512BW-NEXT: sarq $63, %r133853; AVX512BW-NEXT: imulq %r10, %r133854; AVX512BW-NEXT: movq %r10, %rax3855; AVX512BW-NEXT: mulq %rcx3856; AVX512BW-NEXT: movq %rdx, %rbx3857; AVX512BW-NEXT: movq %rax, %r103858; AVX512BW-NEXT: addq %r15, %r103859; AVX512BW-NEXT: adcq %r13, %rbx3860; AVX512BW-NEXT: movq %rbx, %r153861; AVX512BW-NEXT: sarq $63, %r153862; AVX512BW-NEXT: addq %r14, %rbx3863; AVX512BW-NEXT: adcq %rbp, %r153864; AVX512BW-NEXT: movq %r11, %rax3865; AVX512BW-NEXT: imulq %rcx3866; AVX512BW-NEXT: addq %rbx, %rax3867; AVX512BW-NEXT: adcq %r15, %rdx3868; AVX512BW-NEXT: movq %r10, 24(%r12)3869; AVX512BW-NEXT: sarq $63, %r103870; AVX512BW-NEXT: xorq %r10, %rdx3871; AVX512BW-NEXT: xorq %rax, %r103872; AVX512BW-NEXT: orq %rdx, %r103873; AVX512BW-NEXT: setne %al3874; AVX512BW-NEXT: kmovd %eax, %k03875; AVX512BW-NEXT: movq %rsi, %rcx3876; AVX512BW-NEXT: sarq $63, %rcx3877; AVX512BW-NEXT: imulq %r8, %rcx3878; AVX512BW-NEXT: movq %rdi, %rax3879; AVX512BW-NEXT: mulq %r83880; AVX512BW-NEXT: movq %rdx, %r113881; AVX512BW-NEXT: movq %rax, %r103882; AVX512BW-NEXT: movq %rsi, %rax3883; AVX512BW-NEXT: mulq %r83884; AVX512BW-NEXT: movq %rdx, %r83885; AVX512BW-NEXT: movq %rax, %rbx3886; AVX512BW-NEXT: addq %r11, %rbx3887; AVX512BW-NEXT: adcq %rcx, %r83888; AVX512BW-NEXT: movq %r8, %rcx3889; AVX512BW-NEXT: sarq $63, %rcx3890; AVX512BW-NEXT: movq %r9, %r143891; AVX512BW-NEXT: sarq $63, %r143892; AVX512BW-NEXT: imulq %rdi, %r143893; AVX512BW-NEXT: movq %rdi, %rax3894; AVX512BW-NEXT: mulq %r93895; AVX512BW-NEXT: movq %rdx, %rdi3896; AVX512BW-NEXT: movq %rax, %r113897; AVX512BW-NEXT: addq %rbx, %r113898; AVX512BW-NEXT: adcq %r14, %rdi3899; AVX512BW-NEXT: movq %rdi, %rbx3900; AVX512BW-NEXT: sarq $63, %rbx3901; AVX512BW-NEXT: addq %r8, %rdi3902; AVX512BW-NEXT: adcq %rcx, %rbx3903; AVX512BW-NEXT: movq %rsi, %rax3904; AVX512BW-NEXT: imulq %r93905; AVX512BW-NEXT: addq %rdi, %rax3906; AVX512BW-NEXT: adcq %rbx, %rdx3907; AVX512BW-NEXT: movq %r11, 8(%r12)3908; AVX512BW-NEXT: sarq $63, %r113909; AVX512BW-NEXT: xorq %r11, %rdx3910; AVX512BW-NEXT: xorq %rax, %r113911; AVX512BW-NEXT: orq %rdx, %r113912; AVX512BW-NEXT: setne %al3913; AVX512BW-NEXT: andl $1, %eax3914; AVX512BW-NEXT: kmovw %eax, %k13915; AVX512BW-NEXT: kshiftlw $1, %k0, %k03916; AVX512BW-NEXT: korw %k0, %k1, %k13917; AVX512BW-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm03918; AVX512BW-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}3919; AVX512BW-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload3920; AVX512BW-NEXT: movq %rax, 16(%r12)3921; AVX512BW-NEXT: movq %r10, (%r12)3922; AVX512BW-NEXT: popq %rbx3923; AVX512BW-NEXT: popq %r123924; AVX512BW-NEXT: popq %r133925; AVX512BW-NEXT: popq %r143926; AVX512BW-NEXT: popq %r153927; AVX512BW-NEXT: popq %rbp3928; AVX512BW-NEXT: retq3929 %t = call {<2 x i128>, <2 x i1>} @llvm.smul.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)3930 %val = extractvalue {<2 x i128>, <2 x i1>} %t, 03931 %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 13932 %res = sext <2 x i1> %obit to <2 x i32>3933 store <2 x i128> %val, ptr %p23934 ret <2 x i32> %res3935}3936