1647 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE423; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX24; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512VL5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl,+avx512vpopcntdq | FileCheck %s --check-prefixes=AVX512VPOPCNT6 7 8;9; Reductions of per-element ctpop results (count all bits in a vector)10;11 12define i64 @reduce_ctpop_v2i64(<2 x i64> %a0) {13; SSE42-LABEL: reduce_ctpop_v2i64:14; SSE42: # %bb.0:15; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]16; SSE42-NEXT: movdqa %xmm0, %xmm217; SSE42-NEXT: pand %xmm1, %xmm218; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]19; SSE42-NEXT: movdqa %xmm3, %xmm420; SSE42-NEXT: pshufb %xmm2, %xmm421; SSE42-NEXT: psrlw $4, %xmm022; SSE42-NEXT: pand %xmm1, %xmm023; SSE42-NEXT: pshufb %xmm0, %xmm324; SSE42-NEXT: paddb %xmm4, %xmm325; SSE42-NEXT: pxor %xmm0, %xmm026; SSE42-NEXT: psadbw %xmm3, %xmm027; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]28; SSE42-NEXT: paddq %xmm0, %xmm129; SSE42-NEXT: movq %xmm1, %rax30; SSE42-NEXT: retq31;32; AVX2-LABEL: reduce_ctpop_v2i64:33; AVX2: # %bb.0:34; AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]35; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm236; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]37; AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm238; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm039; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm040; AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm041; AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm042; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm143; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm044; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]45; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm046; AVX2-NEXT: vmovq %xmm0, %rax47; AVX2-NEXT: retq48;49; AVX512VL-LABEL: reduce_ctpop_v2i64:50; AVX512VL: # %bb.0:51; AVX512VL-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]52; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm253; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]54; AVX512VL-NEXT: vpshufb %xmm2, %xmm3, %xmm255; AVX512VL-NEXT: vpsrlw $4, %xmm0, %xmm056; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm057; AVX512VL-NEXT: vpshufb %xmm0, %xmm3, %xmm058; AVX512VL-NEXT: vpaddb %xmm2, %xmm0, %xmm059; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm160; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm061; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]62; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm063; AVX512VL-NEXT: vmovq %xmm0, %rax64; AVX512VL-NEXT: retq65;66; AVX512VPOPCNT-LABEL: reduce_ctpop_v2i64:67; AVX512VPOPCNT: # %bb.0:68; AVX512VPOPCNT-NEXT: vpopcntq %xmm0, %xmm069; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]70; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm0, %xmm071; AVX512VPOPCNT-NEXT: vmovq %xmm0, %rax72; AVX512VPOPCNT-NEXT: retq73 %p0 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a0)74 %r0 = tail call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %p0)75 ret i64 %r076}77 78define i32 @reduce_ctpop_v4i32(<4 x i32> %a0) {79; SSE42-LABEL: reduce_ctpop_v4i32:80; SSE42: # %bb.0:81; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]82; SSE42-NEXT: movdqa %xmm0, %xmm283; SSE42-NEXT: pand %xmm1, %xmm284; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]85; SSE42-NEXT: movdqa %xmm3, %xmm486; SSE42-NEXT: pshufb %xmm2, %xmm487; SSE42-NEXT: psrlw $4, %xmm088; SSE42-NEXT: pand %xmm1, %xmm089; SSE42-NEXT: pshufb %xmm0, %xmm390; SSE42-NEXT: paddb %xmm4, %xmm391; SSE42-NEXT: pxor %xmm0, %xmm092; SSE42-NEXT: pmovzxdq {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero93; SSE42-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm0[2],xmm3[3],xmm0[3]94; SSE42-NEXT: psadbw %xmm0, %xmm395; SSE42-NEXT: psadbw %xmm0, %xmm196; SSE42-NEXT: packuswb %xmm3, %xmm197; SSE42-NEXT: packuswb %xmm3, %xmm398; SSE42-NEXT: paddd %xmm1, %xmm399; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]100; SSE42-NEXT: paddd %xmm3, %xmm0101; SSE42-NEXT: movd %xmm0, %eax102; SSE42-NEXT: retq103;104; AVX2-LABEL: reduce_ctpop_v4i32:105; AVX2: # %bb.0:106; AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]107; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2108; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]109; AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2110; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0111; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0112; AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm0113; AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0114; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1115; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]116; AVX2-NEXT: vpsadbw %xmm1, %xmm2, %xmm2117; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero118; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0119; AVX2-NEXT: vpackuswb %xmm2, %xmm0, %xmm0120; AVX2-NEXT: vpackuswb %xmm2, %xmm2, %xmm1121; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0122; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]123; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0124; AVX2-NEXT: vmovd %xmm0, %eax125; AVX2-NEXT: retq126;127; AVX512VL-LABEL: reduce_ctpop_v4i32:128; AVX512VL: # %bb.0:129; AVX512VL-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]130; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm2131; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]132; AVX512VL-NEXT: vpshufb %xmm2, %xmm3, %xmm2133; AVX512VL-NEXT: vpsrlw $4, %xmm0, %xmm0134; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm0135; AVX512VL-NEXT: vpshufb %xmm0, %xmm3, %xmm0136; AVX512VL-NEXT: vpaddb %xmm2, %xmm0, %xmm0137; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1138; AVX512VL-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]139; AVX512VL-NEXT: vpsadbw %xmm1, %xmm2, %xmm2140; AVX512VL-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero141; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0142; AVX512VL-NEXT: vpackuswb %xmm2, %xmm0, %xmm0143; AVX512VL-NEXT: vpmovdb %xmm0, %xmm0144; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0145; AVX512VL-NEXT: vmovd %xmm0, %eax146; AVX512VL-NEXT: retq147;148; AVX512VPOPCNT-LABEL: reduce_ctpop_v4i32:149; AVX512VPOPCNT: # %bb.0:150; AVX512VPOPCNT-NEXT: vpopcntd %xmm0, %xmm0151; AVX512VPOPCNT-NEXT: vpmovdb %xmm0, %xmm0152; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1153; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0154; AVX512VPOPCNT-NEXT: vmovd %xmm0, %eax155; AVX512VPOPCNT-NEXT: retq156 %p0 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %a0)157 %r0 = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %p0)158 ret i32 %r0159}160 161define i16 @reduce_ctpop_v8i16(<8 x i16> %a0) {162; SSE42-LABEL: reduce_ctpop_v8i16:163; SSE42: # %bb.0:164; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]165; SSE42-NEXT: movdqa %xmm0, %xmm2166; SSE42-NEXT: pand %xmm1, %xmm2167; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]168; SSE42-NEXT: movdqa %xmm3, %xmm4169; SSE42-NEXT: pshufb %xmm2, %xmm4170; SSE42-NEXT: psrlw $4, %xmm0171; SSE42-NEXT: pand %xmm1, %xmm0172; SSE42-NEXT: pshufb %xmm0, %xmm3173; SSE42-NEXT: paddb %xmm4, %xmm3174; SSE42-NEXT: movdqa %xmm3, %xmm0175; SSE42-NEXT: psllw $8, %xmm0176; SSE42-NEXT: paddb %xmm3, %xmm0177; SSE42-NEXT: psrlw $8, %xmm0178; SSE42-NEXT: packuswb %xmm0, %xmm0179; SSE42-NEXT: pxor %xmm1, %xmm1180; SSE42-NEXT: psadbw %xmm0, %xmm1181; SSE42-NEXT: movd %xmm1, %eax182; SSE42-NEXT: # kill: def $ax killed $ax killed $eax183; SSE42-NEXT: retq184;185; AVX2-LABEL: reduce_ctpop_v8i16:186; AVX2: # %bb.0:187; AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]188; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2189; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]190; AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2191; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0192; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0193; AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm0194; AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0195; AVX2-NEXT: vpsllw $8, %xmm0, %xmm1196; AVX2-NEXT: vpaddb %xmm0, %xmm1, %xmm0197; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0198; AVX2-NEXT: vpackuswb %xmm0, %xmm0, %xmm0199; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1200; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0201; AVX2-NEXT: vmovd %xmm0, %eax202; AVX2-NEXT: # kill: def $ax killed $ax killed $eax203; AVX2-NEXT: retq204;205; AVX512VL-LABEL: reduce_ctpop_v8i16:206; AVX512VL: # %bb.0:207; AVX512VL-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]208; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm2209; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]210; AVX512VL-NEXT: vpshufb %xmm2, %xmm3, %xmm2211; AVX512VL-NEXT: vpsrlw $4, %xmm0, %xmm0212; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm0213; AVX512VL-NEXT: vpshufb %xmm0, %xmm3, %xmm0214; AVX512VL-NEXT: vpaddb %xmm2, %xmm0, %xmm0215; AVX512VL-NEXT: vpsllw $8, %xmm0, %xmm1216; AVX512VL-NEXT: vpaddb %xmm0, %xmm1, %xmm0217; AVX512VL-NEXT: vpsrlw $8, %xmm0, %xmm0218; AVX512VL-NEXT: vpackuswb %xmm0, %xmm0, %xmm0219; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1220; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0221; AVX512VL-NEXT: vmovd %xmm0, %eax222; AVX512VL-NEXT: # kill: def $ax killed $ax killed $eax223; AVX512VL-NEXT: retq224;225; AVX512VPOPCNT-LABEL: reduce_ctpop_v8i16:226; AVX512VPOPCNT: # %bb.0:227; AVX512VPOPCNT-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero228; AVX512VPOPCNT-NEXT: vpopcntd %ymm0, %ymm0229; AVX512VPOPCNT-NEXT: vpmovdb %ymm0, %xmm0230; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1231; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0232; AVX512VPOPCNT-NEXT: vmovd %xmm0, %eax233; AVX512VPOPCNT-NEXT: # kill: def $ax killed $ax killed $eax234; AVX512VPOPCNT-NEXT: vzeroupper235; AVX512VPOPCNT-NEXT: retq236 %p0 = tail call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %a0)237 %r0 = tail call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %p0)238 ret i16 %r0239}240 241define i8 @reduce_ctpop_v16i8(<16 x i8> %a0) {242; SSE42-LABEL: reduce_ctpop_v16i8:243; SSE42: # %bb.0:244; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]245; SSE42-NEXT: movdqa %xmm0, %xmm2246; SSE42-NEXT: pand %xmm1, %xmm2247; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]248; SSE42-NEXT: movdqa %xmm3, %xmm4249; SSE42-NEXT: pshufb %xmm2, %xmm4250; SSE42-NEXT: psrlw $4, %xmm0251; SSE42-NEXT: pand %xmm1, %xmm0252; SSE42-NEXT: pshufb %xmm0, %xmm3253; SSE42-NEXT: paddb %xmm4, %xmm3254; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]255; SSE42-NEXT: paddb %xmm3, %xmm0256; SSE42-NEXT: pxor %xmm1, %xmm1257; SSE42-NEXT: psadbw %xmm0, %xmm1258; SSE42-NEXT: movd %xmm1, %eax259; SSE42-NEXT: # kill: def $al killed $al killed $eax260; SSE42-NEXT: retq261;262; AVX2-LABEL: reduce_ctpop_v16i8:263; AVX2: # %bb.0:264; AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]265; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2266; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]267; AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2268; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0269; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0270; AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm0271; AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0272; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]273; AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0274; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1275; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0276; AVX2-NEXT: vmovd %xmm0, %eax277; AVX2-NEXT: # kill: def $al killed $al killed $eax278; AVX2-NEXT: retq279;280; AVX512VL-LABEL: reduce_ctpop_v16i8:281; AVX512VL: # %bb.0:282; AVX512VL-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]283; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm2284; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]285; AVX512VL-NEXT: vpshufb %xmm2, %xmm3, %xmm2286; AVX512VL-NEXT: vpsrlw $4, %xmm0, %xmm0287; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm0288; AVX512VL-NEXT: vpshufb %xmm0, %xmm3, %xmm0289; AVX512VL-NEXT: vpaddb %xmm2, %xmm0, %xmm0290; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]291; AVX512VL-NEXT: vpaddb %xmm1, %xmm0, %xmm0292; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1293; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0294; AVX512VL-NEXT: vmovd %xmm0, %eax295; AVX512VL-NEXT: # kill: def $al killed $al killed $eax296; AVX512VL-NEXT: retq297;298; AVX512VPOPCNT-LABEL: reduce_ctpop_v16i8:299; AVX512VPOPCNT: # %bb.0:300; AVX512VPOPCNT-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero301; AVX512VPOPCNT-NEXT: vpopcntd %zmm0, %zmm0302; AVX512VPOPCNT-NEXT: vpmovdb %zmm0, %xmm0303; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]304; AVX512VPOPCNT-NEXT: vpaddb %xmm1, %xmm0, %xmm0305; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1306; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0307; AVX512VPOPCNT-NEXT: vmovd %xmm0, %eax308; AVX512VPOPCNT-NEXT: # kill: def $al killed $al killed $eax309; AVX512VPOPCNT-NEXT: vzeroupper310; AVX512VPOPCNT-NEXT: retq311 %p0 = tail call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> %a0)312 %r0 = tail call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %p0)313 ret i8 %r0314}315 316define i64 @reduce_ctpop_v4i64(<4 x i64> %a0) {317; SSE42-LABEL: reduce_ctpop_v4i64:318; SSE42: # %bb.0:319; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]320; SSE42-NEXT: movdqa %xmm1, %xmm3321; SSE42-NEXT: pand %xmm2, %xmm3322; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]323; SSE42-NEXT: movdqa %xmm4, %xmm5324; SSE42-NEXT: pshufb %xmm3, %xmm5325; SSE42-NEXT: psrlw $4, %xmm1326; SSE42-NEXT: pand %xmm2, %xmm1327; SSE42-NEXT: movdqa %xmm4, %xmm3328; SSE42-NEXT: pshufb %xmm1, %xmm3329; SSE42-NEXT: paddb %xmm5, %xmm3330; SSE42-NEXT: movdqa %xmm0, %xmm1331; SSE42-NEXT: pand %xmm2, %xmm1332; SSE42-NEXT: movdqa %xmm4, %xmm5333; SSE42-NEXT: pshufb %xmm1, %xmm5334; SSE42-NEXT: psrlw $4, %xmm0335; SSE42-NEXT: pand %xmm2, %xmm0336; SSE42-NEXT: pshufb %xmm0, %xmm4337; SSE42-NEXT: paddb %xmm5, %xmm4338; SSE42-NEXT: paddb %xmm3, %xmm4339; SSE42-NEXT: pxor %xmm0, %xmm0340; SSE42-NEXT: psadbw %xmm4, %xmm0341; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]342; SSE42-NEXT: paddq %xmm0, %xmm1343; SSE42-NEXT: movq %xmm1, %rax344; SSE42-NEXT: retq345;346; AVX2-LABEL: reduce_ctpop_v4i64:347; AVX2: # %bb.0:348; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]349; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm2350; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]351; AVX2-NEXT: # ymm3 = mem[0,1,0,1]352; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm2353; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0354; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0355; AVX2-NEXT: vpshufb %ymm0, %ymm3, %ymm0356; AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0357; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1358; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0359; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1360; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0361; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]362; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0363; AVX2-NEXT: vmovq %xmm0, %rax364; AVX2-NEXT: vzeroupper365; AVX2-NEXT: retq366;367; AVX512VL-LABEL: reduce_ctpop_v4i64:368; AVX512VL: # %bb.0:369; AVX512VL-NEXT: vpbroadcastb {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]370; AVX512VL-NEXT: vpand %ymm1, %ymm0, %ymm2371; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]372; AVX512VL-NEXT: # ymm3 = mem[0,1,0,1]373; AVX512VL-NEXT: vpshufb %ymm2, %ymm3, %ymm2374; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0375; AVX512VL-NEXT: vpand %ymm1, %ymm0, %ymm0376; AVX512VL-NEXT: vpshufb %ymm0, %ymm3, %ymm0377; AVX512VL-NEXT: vpaddb %ymm2, %ymm0, %ymm0378; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1379; AVX512VL-NEXT: vpsadbw %ymm1, %ymm0, %ymm0380; AVX512VL-NEXT: vpmovqb %ymm0, %xmm0381; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1382; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0383; AVX512VL-NEXT: vmovq %xmm0, %rax384; AVX512VL-NEXT: vzeroupper385; AVX512VL-NEXT: retq386;387; AVX512VPOPCNT-LABEL: reduce_ctpop_v4i64:388; AVX512VPOPCNT: # %bb.0:389; AVX512VPOPCNT-NEXT: vpopcntq %ymm0, %ymm0390; AVX512VPOPCNT-NEXT: vpmovqb %ymm0, %xmm0391; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1392; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0393; AVX512VPOPCNT-NEXT: vmovq %xmm0, %rax394; AVX512VPOPCNT-NEXT: vzeroupper395; AVX512VPOPCNT-NEXT: retq396 %p0 = tail call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a0)397 %r0 = tail call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %p0)398 ret i64 %r0399}400 401define i32 @reduce_ctpop_v8i32(<8 x i32> %a0) {402; SSE42-LABEL: reduce_ctpop_v8i32:403; SSE42: # %bb.0:404; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]405; SSE42-NEXT: movdqa %xmm1, %xmm4406; SSE42-NEXT: pand %xmm3, %xmm4407; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]408; SSE42-NEXT: movdqa %xmm2, %xmm5409; SSE42-NEXT: pshufb %xmm4, %xmm5410; SSE42-NEXT: psrlw $4, %xmm1411; SSE42-NEXT: pand %xmm3, %xmm1412; SSE42-NEXT: movdqa %xmm2, %xmm4413; SSE42-NEXT: pshufb %xmm1, %xmm4414; SSE42-NEXT: paddb %xmm5, %xmm4415; SSE42-NEXT: pxor %xmm1, %xmm1416; SSE42-NEXT: pmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero417; SSE42-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]418; SSE42-NEXT: psadbw %xmm1, %xmm4419; SSE42-NEXT: psadbw %xmm1, %xmm5420; SSE42-NEXT: packuswb %xmm4, %xmm5421; SSE42-NEXT: movdqa %xmm0, %xmm4422; SSE42-NEXT: pand %xmm3, %xmm4423; SSE42-NEXT: movdqa %xmm2, %xmm6424; SSE42-NEXT: pshufb %xmm4, %xmm6425; SSE42-NEXT: psrlw $4, %xmm0426; SSE42-NEXT: pand %xmm3, %xmm0427; SSE42-NEXT: pshufb %xmm0, %xmm2428; SSE42-NEXT: paddb %xmm6, %xmm2429; SSE42-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero430; SSE42-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]431; SSE42-NEXT: psadbw %xmm1, %xmm2432; SSE42-NEXT: psadbw %xmm1, %xmm0433; SSE42-NEXT: packuswb %xmm2, %xmm0434; SSE42-NEXT: paddd %xmm5, %xmm0435; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]436; SSE42-NEXT: paddd %xmm0, %xmm1437; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]438; SSE42-NEXT: paddd %xmm1, %xmm0439; SSE42-NEXT: movd %xmm0, %eax440; SSE42-NEXT: retq441;442; AVX2-LABEL: reduce_ctpop_v8i32:443; AVX2: # %bb.0:444; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]445; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm2446; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]447; AVX2-NEXT: # ymm3 = mem[0,1,0,1]448; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm2449; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0450; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0451; AVX2-NEXT: vpshufb %ymm0, %ymm3, %ymm0452; AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0453; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1454; AVX2-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]455; AVX2-NEXT: vpsadbw %ymm1, %ymm2, %ymm2456; AVX2-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]457; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0458; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0459; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1460; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0461; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]462; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0463; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]464; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0465; AVX2-NEXT: vmovd %xmm0, %eax466; AVX2-NEXT: vzeroupper467; AVX2-NEXT: retq468;469; AVX512VL-LABEL: reduce_ctpop_v8i32:470; AVX512VL: # %bb.0:471; AVX512VL-NEXT: vpbroadcastb {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]472; AVX512VL-NEXT: vpand %ymm1, %ymm0, %ymm2473; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]474; AVX512VL-NEXT: # ymm3 = mem[0,1,0,1]475; AVX512VL-NEXT: vpshufb %ymm2, %ymm3, %ymm2476; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0477; AVX512VL-NEXT: vpand %ymm1, %ymm0, %ymm0478; AVX512VL-NEXT: vpshufb %ymm0, %ymm3, %ymm0479; AVX512VL-NEXT: vpaddb %ymm2, %ymm0, %ymm0480; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1481; AVX512VL-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]482; AVX512VL-NEXT: vpsadbw %ymm1, %ymm2, %ymm2483; AVX512VL-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]484; AVX512VL-NEXT: vpsadbw %ymm1, %ymm0, %ymm0485; AVX512VL-NEXT: vpackuswb %ymm2, %ymm0, %ymm0486; AVX512VL-NEXT: vpmovdb %ymm0, %xmm0487; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1488; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0489; AVX512VL-NEXT: vmovd %xmm0, %eax490; AVX512VL-NEXT: vzeroupper491; AVX512VL-NEXT: retq492;493; AVX512VPOPCNT-LABEL: reduce_ctpop_v8i32:494; AVX512VPOPCNT: # %bb.0:495; AVX512VPOPCNT-NEXT: vpopcntd %ymm0, %ymm0496; AVX512VPOPCNT-NEXT: vpmovdb %ymm0, %xmm0497; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1498; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0499; AVX512VPOPCNT-NEXT: vmovd %xmm0, %eax500; AVX512VPOPCNT-NEXT: vzeroupper501; AVX512VPOPCNT-NEXT: retq502 %p0 = tail call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %a0)503 %r0 = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %p0)504 ret i32 %r0505}506 507define i64 @reduce_ctpop_v8i64(<8 x i64> %a0) {508; SSE42-LABEL: reduce_ctpop_v8i64:509; SSE42: # %bb.0:510; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]511; SSE42-NEXT: movdqa %xmm2, %xmm6512; SSE42-NEXT: pand %xmm5, %xmm6513; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]514; SSE42-NEXT: movdqa %xmm4, %xmm7515; SSE42-NEXT: pshufb %xmm6, %xmm7516; SSE42-NEXT: psrlw $4, %xmm2517; SSE42-NEXT: pand %xmm5, %xmm2518; SSE42-NEXT: movdqa %xmm4, %xmm6519; SSE42-NEXT: pshufb %xmm2, %xmm6520; SSE42-NEXT: paddb %xmm7, %xmm6521; SSE42-NEXT: movdqa %xmm0, %xmm2522; SSE42-NEXT: pand %xmm5, %xmm2523; SSE42-NEXT: movdqa %xmm4, %xmm7524; SSE42-NEXT: pshufb %xmm2, %xmm7525; SSE42-NEXT: psrlw $4, %xmm0526; SSE42-NEXT: pand %xmm5, %xmm0527; SSE42-NEXT: movdqa %xmm4, %xmm2528; SSE42-NEXT: pshufb %xmm0, %xmm2529; SSE42-NEXT: paddb %xmm7, %xmm2530; SSE42-NEXT: paddb %xmm6, %xmm2531; SSE42-NEXT: movdqa %xmm3, %xmm0532; SSE42-NEXT: pand %xmm5, %xmm0533; SSE42-NEXT: movdqa %xmm4, %xmm6534; SSE42-NEXT: pshufb %xmm0, %xmm6535; SSE42-NEXT: psrlw $4, %xmm3536; SSE42-NEXT: pand %xmm5, %xmm3537; SSE42-NEXT: movdqa %xmm4, %xmm0538; SSE42-NEXT: pshufb %xmm3, %xmm0539; SSE42-NEXT: paddb %xmm6, %xmm0540; SSE42-NEXT: movdqa %xmm1, %xmm3541; SSE42-NEXT: pand %xmm5, %xmm3542; SSE42-NEXT: movdqa %xmm4, %xmm6543; SSE42-NEXT: pshufb %xmm3, %xmm6544; SSE42-NEXT: psrlw $4, %xmm1545; SSE42-NEXT: pand %xmm5, %xmm1546; SSE42-NEXT: pshufb %xmm1, %xmm4547; SSE42-NEXT: paddb %xmm6, %xmm4548; SSE42-NEXT: paddb %xmm0, %xmm4549; SSE42-NEXT: paddb %xmm2, %xmm4550; SSE42-NEXT: pxor %xmm0, %xmm0551; SSE42-NEXT: psadbw %xmm4, %xmm0552; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]553; SSE42-NEXT: paddq %xmm0, %xmm1554; SSE42-NEXT: movq %xmm1, %rax555; SSE42-NEXT: retq556;557; AVX2-LABEL: reduce_ctpop_v8i64:558; AVX2: # %bb.0:559; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]560; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm3561; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]562; AVX2-NEXT: # ymm4 = mem[0,1,0,1]563; AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm3564; AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1565; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1566; AVX2-NEXT: vpshufb %ymm1, %ymm4, %ymm1567; AVX2-NEXT: vpaddb %ymm3, %ymm1, %ymm1568; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3569; AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm3570; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0571; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0572; AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0573; AVX2-NEXT: vpaddb %ymm3, %ymm0, %ymm0574; AVX2-NEXT: vpaddb %ymm1, %ymm0, %ymm0575; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1576; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0577; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1578; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0579; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]580; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0581; AVX2-NEXT: vmovq %xmm0, %rax582; AVX2-NEXT: vzeroupper583; AVX2-NEXT: retq584;585; AVX512VL-LABEL: reduce_ctpop_v8i64:586; AVX512VL: # %bb.0:587; AVX512VL-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]588; AVX512VL-NEXT: vpandq %zmm1, %zmm0, %zmm2589; AVX512VL-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]590; AVX512VL-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]591; AVX512VL-NEXT: vpshufb %zmm2, %zmm3, %zmm2592; AVX512VL-NEXT: vpsrlw $4, %zmm0, %zmm0593; AVX512VL-NEXT: vpandq %zmm1, %zmm0, %zmm0594; AVX512VL-NEXT: vpshufb %zmm0, %zmm3, %zmm0595; AVX512VL-NEXT: vpaddb %zmm2, %zmm0, %zmm0596; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1597; AVX512VL-NEXT: vpsadbw %zmm1, %zmm0, %zmm0598; AVX512VL-NEXT: vpmovqb %zmm0, %xmm0599; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1600; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0601; AVX512VL-NEXT: vmovq %xmm0, %rax602; AVX512VL-NEXT: vzeroupper603; AVX512VL-NEXT: retq604;605; AVX512VPOPCNT-LABEL: reduce_ctpop_v8i64:606; AVX512VPOPCNT: # %bb.0:607; AVX512VPOPCNT-NEXT: vpopcntq %zmm0, %zmm0608; AVX512VPOPCNT-NEXT: vpmovqb %zmm0, %xmm0609; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1610; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0611; AVX512VPOPCNT-NEXT: vmovq %xmm0, %rax612; AVX512VPOPCNT-NEXT: vzeroupper613; AVX512VPOPCNT-NEXT: retq614 %p0 = tail call <8 x i64> @llvm.ctpop.v8i64(<8 x i64> %a0)615 %r0 = tail call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %p0)616 ret i64 %r0617}618 619define i32 @reduce_ctpop_v16i32(<16 x i32> %a0) {620; SSE42-LABEL: reduce_ctpop_v16i32:621; SSE42: # %bb.0:622; SSE42-NEXT: movdqa {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]623; SSE42-NEXT: movdqa %xmm2, %xmm6624; SSE42-NEXT: pand %xmm5, %xmm6625; SSE42-NEXT: movdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]626; SSE42-NEXT: movdqa %xmm4, %xmm7627; SSE42-NEXT: pshufb %xmm6, %xmm7628; SSE42-NEXT: psrlw $4, %xmm2629; SSE42-NEXT: pand %xmm5, %xmm2630; SSE42-NEXT: movdqa %xmm4, %xmm6631; SSE42-NEXT: pshufb %xmm2, %xmm6632; SSE42-NEXT: paddb %xmm7, %xmm6633; SSE42-NEXT: pxor %xmm2, %xmm2634; SSE42-NEXT: pmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero635; SSE42-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm2[2],xmm6[3],xmm2[3]636; SSE42-NEXT: psadbw %xmm2, %xmm6637; SSE42-NEXT: psadbw %xmm2, %xmm7638; SSE42-NEXT: packuswb %xmm6, %xmm7639; SSE42-NEXT: movdqa %xmm0, %xmm6640; SSE42-NEXT: pand %xmm5, %xmm6641; SSE42-NEXT: movdqa %xmm4, %xmm8642; SSE42-NEXT: pshufb %xmm6, %xmm8643; SSE42-NEXT: psrlw $4, %xmm0644; SSE42-NEXT: pand %xmm5, %xmm0645; SSE42-NEXT: movdqa %xmm4, %xmm6646; SSE42-NEXT: pshufb %xmm0, %xmm6647; SSE42-NEXT: paddb %xmm8, %xmm6648; SSE42-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm6[0],zero,xmm6[1],zero649; SSE42-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm2[2],xmm6[3],xmm2[3]650; SSE42-NEXT: psadbw %xmm2, %xmm6651; SSE42-NEXT: psadbw %xmm2, %xmm0652; SSE42-NEXT: packuswb %xmm6, %xmm0653; SSE42-NEXT: paddd %xmm7, %xmm0654; SSE42-NEXT: movdqa %xmm3, %xmm6655; SSE42-NEXT: pand %xmm5, %xmm6656; SSE42-NEXT: movdqa %xmm4, %xmm7657; SSE42-NEXT: pshufb %xmm6, %xmm7658; SSE42-NEXT: psrlw $4, %xmm3659; SSE42-NEXT: pand %xmm5, %xmm3660; SSE42-NEXT: movdqa %xmm4, %xmm6661; SSE42-NEXT: pshufb %xmm3, %xmm6662; SSE42-NEXT: paddb %xmm7, %xmm6663; SSE42-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm6[0],zero,xmm6[1],zero664; SSE42-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm2[2],xmm6[3],xmm2[3]665; SSE42-NEXT: psadbw %xmm2, %xmm6666; SSE42-NEXT: psadbw %xmm2, %xmm3667; SSE42-NEXT: packuswb %xmm6, %xmm3668; SSE42-NEXT: movdqa %xmm1, %xmm6669; SSE42-NEXT: pand %xmm5, %xmm6670; SSE42-NEXT: movdqa %xmm4, %xmm7671; SSE42-NEXT: pshufb %xmm6, %xmm7672; SSE42-NEXT: psrlw $4, %xmm1673; SSE42-NEXT: pand %xmm5, %xmm1674; SSE42-NEXT: pshufb %xmm1, %xmm4675; SSE42-NEXT: paddb %xmm7, %xmm4676; SSE42-NEXT: pmovzxdq {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero677; SSE42-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]678; SSE42-NEXT: psadbw %xmm2, %xmm4679; SSE42-NEXT: psadbw %xmm2, %xmm1680; SSE42-NEXT: packuswb %xmm4, %xmm1681; SSE42-NEXT: paddd %xmm3, %xmm1682; SSE42-NEXT: paddd %xmm0, %xmm1683; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]684; SSE42-NEXT: paddd %xmm1, %xmm0685; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]686; SSE42-NEXT: paddd %xmm0, %xmm1687; SSE42-NEXT: movd %xmm1, %eax688; SSE42-NEXT: retq689;690; AVX2-LABEL: reduce_ctpop_v16i32:691; AVX2: # %bb.0:692; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]693; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm3694; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]695; AVX2-NEXT: # ymm4 = mem[0,1,0,1]696; AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm3697; AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1698; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1699; AVX2-NEXT: vpshufb %ymm1, %ymm4, %ymm1700; AVX2-NEXT: vpaddb %ymm3, %ymm1, %ymm1701; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3702; AVX2-NEXT: vpunpckhdq {{.*#+}} ymm5 = ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[6],ymm3[6],ymm1[7],ymm3[7]703; AVX2-NEXT: vpsadbw %ymm3, %ymm5, %ymm5704; AVX2-NEXT: vpunpckldq {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[4],ymm3[4],ymm1[5],ymm3[5]705; AVX2-NEXT: vpsadbw %ymm3, %ymm1, %ymm1706; AVX2-NEXT: vpackuswb %ymm5, %ymm1, %ymm1707; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm5708; AVX2-NEXT: vpshufb %ymm5, %ymm4, %ymm5709; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0710; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0711; AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0712; AVX2-NEXT: vpaddb %ymm5, %ymm0, %ymm0713; AVX2-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[6],ymm3[6],ymm0[7],ymm3[7]714; AVX2-NEXT: vpsadbw %ymm3, %ymm2, %ymm2715; AVX2-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[4],ymm3[4],ymm0[5],ymm3[5]716; AVX2-NEXT: vpsadbw %ymm3, %ymm0, %ymm0717; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0718; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0719; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1720; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0721; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]722; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0723; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]724; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0725; AVX2-NEXT: vmovd %xmm0, %eax726; AVX2-NEXT: vzeroupper727; AVX2-NEXT: retq728;729; AVX512VL-LABEL: reduce_ctpop_v16i32:730; AVX512VL: # %bb.0:731; AVX512VL-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]732; AVX512VL-NEXT: vpandq %zmm1, %zmm0, %zmm2733; AVX512VL-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]734; AVX512VL-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]735; AVX512VL-NEXT: vpshufb %zmm2, %zmm3, %zmm2736; AVX512VL-NEXT: vpsrlw $4, %zmm0, %zmm0737; AVX512VL-NEXT: vpandq %zmm1, %zmm0, %zmm0738; AVX512VL-NEXT: vpshufb %zmm0, %zmm3, %zmm0739; AVX512VL-NEXT: vpaddb %zmm2, %zmm0, %zmm0740; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1741; AVX512VL-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]742; AVX512VL-NEXT: vpsadbw %zmm1, %zmm2, %zmm2743; AVX512VL-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]744; AVX512VL-NEXT: vpsadbw %zmm1, %zmm0, %zmm0745; AVX512VL-NEXT: vpackuswb %zmm2, %zmm0, %zmm0746; AVX512VL-NEXT: vpmovdb %zmm0, %xmm0747; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1748; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0749; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]750; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0751; AVX512VL-NEXT: vmovd %xmm0, %eax752; AVX512VL-NEXT: vzeroupper753; AVX512VL-NEXT: retq754;755; AVX512VPOPCNT-LABEL: reduce_ctpop_v16i32:756; AVX512VPOPCNT: # %bb.0:757; AVX512VPOPCNT-NEXT: vpopcntd %zmm0, %zmm0758; AVX512VPOPCNT-NEXT: vpmovdb %zmm0, %xmm0759; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1760; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0761; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]762; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm0, %xmm0763; AVX512VPOPCNT-NEXT: vmovd %xmm0, %eax764; AVX512VPOPCNT-NEXT: vzeroupper765; AVX512VPOPCNT-NEXT: retq766 %p0 = tail call <16 x i32> @llvm.ctpop.v16i32(<16 x i32> %a0)767 %r0 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %p0)768 ret i32 %r0769}770 771define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) {772; SSE42-LABEL: reduce_ctpop_v16i64:773; SSE42: # %bb.0:774; SSE42-NEXT: movdqa {{.*#+}} xmm9 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]775; SSE42-NEXT: movdqa %xmm5, %xmm10776; SSE42-NEXT: pand %xmm9, %xmm10777; SSE42-NEXT: movdqa {{.*#+}} xmm8 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]778; SSE42-NEXT: movdqa %xmm8, %xmm11779; SSE42-NEXT: pshufb %xmm10, %xmm11780; SSE42-NEXT: psrlw $4, %xmm5781; SSE42-NEXT: pand %xmm9, %xmm5782; SSE42-NEXT: movdqa %xmm8, %xmm10783; SSE42-NEXT: pshufb %xmm5, %xmm10784; SSE42-NEXT: paddb %xmm11, %xmm10785; SSE42-NEXT: movdqa %xmm1, %xmm5786; SSE42-NEXT: pand %xmm9, %xmm5787; SSE42-NEXT: movdqa %xmm8, %xmm11788; SSE42-NEXT: pshufb %xmm5, %xmm11789; SSE42-NEXT: psrlw $4, %xmm1790; SSE42-NEXT: pand %xmm9, %xmm1791; SSE42-NEXT: movdqa %xmm8, %xmm5792; SSE42-NEXT: pshufb %xmm1, %xmm5793; SSE42-NEXT: paddb %xmm11, %xmm5794; SSE42-NEXT: paddb %xmm10, %xmm5795; SSE42-NEXT: movdqa %xmm7, %xmm1796; SSE42-NEXT: pand %xmm9, %xmm1797; SSE42-NEXT: movdqa %xmm8, %xmm10798; SSE42-NEXT: pshufb %xmm1, %xmm10799; SSE42-NEXT: psrlw $4, %xmm7800; SSE42-NEXT: pand %xmm9, %xmm7801; SSE42-NEXT: movdqa %xmm8, %xmm11802; SSE42-NEXT: pshufb %xmm7, %xmm11803; SSE42-NEXT: paddb %xmm10, %xmm11804; SSE42-NEXT: movdqa %xmm3, %xmm1805; SSE42-NEXT: pand %xmm9, %xmm1806; SSE42-NEXT: movdqa %xmm8, %xmm7807; SSE42-NEXT: pshufb %xmm1, %xmm7808; SSE42-NEXT: psrlw $4, %xmm3809; SSE42-NEXT: pand %xmm9, %xmm3810; SSE42-NEXT: movdqa %xmm8, %xmm1811; SSE42-NEXT: pshufb %xmm3, %xmm1812; SSE42-NEXT: paddb %xmm7, %xmm1813; SSE42-NEXT: paddb %xmm11, %xmm1814; SSE42-NEXT: paddb %xmm5, %xmm1815; SSE42-NEXT: movdqa %xmm4, %xmm3816; SSE42-NEXT: pand %xmm9, %xmm3817; SSE42-NEXT: movdqa %xmm8, %xmm5818; SSE42-NEXT: pshufb %xmm3, %xmm5819; SSE42-NEXT: psrlw $4, %xmm4820; SSE42-NEXT: pand %xmm9, %xmm4821; SSE42-NEXT: movdqa %xmm8, %xmm7822; SSE42-NEXT: pshufb %xmm4, %xmm7823; SSE42-NEXT: paddb %xmm5, %xmm7824; SSE42-NEXT: movdqa %xmm0, %xmm3825; SSE42-NEXT: pand %xmm9, %xmm3826; SSE42-NEXT: movdqa %xmm8, %xmm4827; SSE42-NEXT: pshufb %xmm3, %xmm4828; SSE42-NEXT: psrlw $4, %xmm0829; SSE42-NEXT: pand %xmm9, %xmm0830; SSE42-NEXT: movdqa %xmm8, %xmm3831; SSE42-NEXT: pshufb %xmm0, %xmm3832; SSE42-NEXT: paddb %xmm4, %xmm3833; SSE42-NEXT: paddb %xmm7, %xmm3834; SSE42-NEXT: movdqa %xmm6, %xmm0835; SSE42-NEXT: pand %xmm9, %xmm0836; SSE42-NEXT: movdqa %xmm8, %xmm4837; SSE42-NEXT: pshufb %xmm0, %xmm4838; SSE42-NEXT: psrlw $4, %xmm6839; SSE42-NEXT: pand %xmm9, %xmm6840; SSE42-NEXT: movdqa %xmm8, %xmm0841; SSE42-NEXT: pshufb %xmm6, %xmm0842; SSE42-NEXT: paddb %xmm4, %xmm0843; SSE42-NEXT: movdqa %xmm2, %xmm4844; SSE42-NEXT: pand %xmm9, %xmm4845; SSE42-NEXT: movdqa %xmm8, %xmm5846; SSE42-NEXT: pshufb %xmm4, %xmm5847; SSE42-NEXT: psrlw $4, %xmm2848; SSE42-NEXT: pand %xmm9, %xmm2849; SSE42-NEXT: pshufb %xmm2, %xmm8850; SSE42-NEXT: paddb %xmm5, %xmm8851; SSE42-NEXT: paddb %xmm0, %xmm8852; SSE42-NEXT: paddb %xmm3, %xmm8853; SSE42-NEXT: paddb %xmm1, %xmm8854; SSE42-NEXT: pxor %xmm0, %xmm0855; SSE42-NEXT: psadbw %xmm8, %xmm0856; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]857; SSE42-NEXT: paddq %xmm0, %xmm1858; SSE42-NEXT: movq %xmm1, %rax859; SSE42-NEXT: retq860;861; AVX2-LABEL: reduce_ctpop_v16i64:862; AVX2: # %bb.0:863; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]864; AVX2-NEXT: vpand %ymm4, %ymm2, %ymm5865; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]866; AVX2-NEXT: # ymm6 = mem[0,1,0,1]867; AVX2-NEXT: vpshufb %ymm5, %ymm6, %ymm5868; AVX2-NEXT: vpsrlw $4, %ymm2, %ymm2869; AVX2-NEXT: vpand %ymm4, %ymm2, %ymm2870; AVX2-NEXT: vpshufb %ymm2, %ymm6, %ymm2871; AVX2-NEXT: vpaddb %ymm5, %ymm2, %ymm2872; AVX2-NEXT: vpand %ymm4, %ymm0, %ymm5873; AVX2-NEXT: vpshufb %ymm5, %ymm6, %ymm5874; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0875; AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0876; AVX2-NEXT: vpshufb %ymm0, %ymm6, %ymm0877; AVX2-NEXT: vpaddb %ymm5, %ymm0, %ymm0878; AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0879; AVX2-NEXT: vpand %ymm4, %ymm3, %ymm2880; AVX2-NEXT: vpshufb %ymm2, %ymm6, %ymm2881; AVX2-NEXT: vpsrlw $4, %ymm3, %ymm3882; AVX2-NEXT: vpand %ymm4, %ymm3, %ymm3883; AVX2-NEXT: vpshufb %ymm3, %ymm6, %ymm3884; AVX2-NEXT: vpaddb %ymm2, %ymm3, %ymm2885; AVX2-NEXT: vpand %ymm4, %ymm1, %ymm3886; AVX2-NEXT: vpshufb %ymm3, %ymm6, %ymm3887; AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1888; AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1889; AVX2-NEXT: vpshufb %ymm1, %ymm6, %ymm1890; AVX2-NEXT: vpaddb %ymm3, %ymm1, %ymm1891; AVX2-NEXT: vpaddb %ymm2, %ymm1, %ymm1892; AVX2-NEXT: vpaddb %ymm1, %ymm0, %ymm0893; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1894; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0895; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1896; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0897; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]898; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0899; AVX2-NEXT: vmovq %xmm0, %rax900; AVX2-NEXT: vzeroupper901; AVX2-NEXT: retq902;903; AVX512VL-LABEL: reduce_ctpop_v16i64:904; AVX512VL: # %bb.0:905; AVX512VL-NEXT: vpbroadcastb {{.*#+}} zmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]906; AVX512VL-NEXT: vpandq %zmm2, %zmm0, %zmm3907; AVX512VL-NEXT: vbroadcasti32x4 {{.*#+}} zmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]908; AVX512VL-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]909; AVX512VL-NEXT: vpshufb %zmm3, %zmm4, %zmm3910; AVX512VL-NEXT: vpsrlw $4, %zmm0, %zmm0911; AVX512VL-NEXT: vpandq %zmm2, %zmm0, %zmm0912; AVX512VL-NEXT: vpshufb %zmm0, %zmm4, %zmm0913; AVX512VL-NEXT: vpaddb %zmm3, %zmm0, %zmm0914; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3915; AVX512VL-NEXT: vpsadbw %zmm3, %zmm0, %zmm0916; AVX512VL-NEXT: vpandq %zmm2, %zmm1, %zmm5917; AVX512VL-NEXT: vpshufb %zmm5, %zmm4, %zmm5918; AVX512VL-NEXT: vpsrlw $4, %zmm1, %zmm1919; AVX512VL-NEXT: vpandq %zmm2, %zmm1, %zmm1920; AVX512VL-NEXT: vpshufb %zmm1, %zmm4, %zmm1921; AVX512VL-NEXT: vpaddb %zmm5, %zmm1, %zmm1922; AVX512VL-NEXT: vpsadbw %zmm3, %zmm1, %zmm1923; AVX512VL-NEXT: vpmovqb %zmm1, %xmm1924; AVX512VL-NEXT: vpmovqb %zmm0, %xmm0925; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]926; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1927; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0928; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]929; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0930; AVX512VL-NEXT: vmovq %xmm0, %rax931; AVX512VL-NEXT: vzeroupper932; AVX512VL-NEXT: retq933;934; AVX512VPOPCNT-LABEL: reduce_ctpop_v16i64:935; AVX512VPOPCNT: # %bb.0:936; AVX512VPOPCNT-NEXT: vpopcntq %zmm0, %zmm0937; AVX512VPOPCNT-NEXT: vpopcntq %zmm1, %zmm1938; AVX512VPOPCNT-NEXT: vpmovqb %zmm1, %xmm1939; AVX512VPOPCNT-NEXT: vpmovqb %zmm0, %xmm0940; AVX512VPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]941; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1942; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0943; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]944; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm0, %xmm0945; AVX512VPOPCNT-NEXT: vmovq %xmm0, %rax946; AVX512VPOPCNT-NEXT: vzeroupper947; AVX512VPOPCNT-NEXT: retq948 %p0 = tail call <16 x i64> @llvm.ctpop.v16i64(<16 x i64> %a0)949 %r0 = tail call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %p0)950 ret i64 %r0951}952 953;954; Vector of reductions of per-element ctpop results (create vector of each count all bits in each vector)955;956 957define <4 x i64> @reduce_ctpop_v4i64_buildvector_v4i64(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> %a2, <4 x i64> %a3) nounwind {958; SSE42-LABEL: reduce_ctpop_v4i64_buildvector_v4i64:959; SSE42: # %bb.0:960; SSE42-NEXT: movdqa %xmm0, %xmm8961; SSE42-NEXT: movdqa {{.*#+}} xmm10 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]962; SSE42-NEXT: movdqa %xmm1, %xmm0963; SSE42-NEXT: pand %xmm10, %xmm0964; SSE42-NEXT: movdqa {{.*#+}} xmm9 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]965; SSE42-NEXT: movdqa %xmm9, %xmm11966; SSE42-NEXT: pshufb %xmm0, %xmm11967; SSE42-NEXT: psrlw $4, %xmm1968; SSE42-NEXT: pand %xmm10, %xmm1969; SSE42-NEXT: movdqa %xmm9, %xmm12970; SSE42-NEXT: pshufb %xmm1, %xmm12971; SSE42-NEXT: paddb %xmm11, %xmm12972; SSE42-NEXT: movdqa %xmm8, %xmm0973; SSE42-NEXT: pand %xmm10, %xmm0974; SSE42-NEXT: movdqa %xmm9, %xmm1975; SSE42-NEXT: pshufb %xmm0, %xmm1976; SSE42-NEXT: psrlw $4, %xmm8977; SSE42-NEXT: pand %xmm10, %xmm8978; SSE42-NEXT: movdqa %xmm9, %xmm0979; SSE42-NEXT: pshufb %xmm8, %xmm0980; SSE42-NEXT: paddb %xmm1, %xmm0981; SSE42-NEXT: paddb %xmm12, %xmm0982; SSE42-NEXT: movdqa %xmm3, %xmm1983; SSE42-NEXT: pand %xmm10, %xmm1984; SSE42-NEXT: movdqa %xmm9, %xmm8985; SSE42-NEXT: pshufb %xmm1, %xmm8986; SSE42-NEXT: psrlw $4, %xmm3987; SSE42-NEXT: pand %xmm10, %xmm3988; SSE42-NEXT: movdqa %xmm9, %xmm1989; SSE42-NEXT: pshufb %xmm3, %xmm1990; SSE42-NEXT: paddb %xmm8, %xmm1991; SSE42-NEXT: movdqa %xmm2, %xmm3992; SSE42-NEXT: pand %xmm10, %xmm3993; SSE42-NEXT: movdqa %xmm9, %xmm8994; SSE42-NEXT: pshufb %xmm3, %xmm8995; SSE42-NEXT: psrlw $4, %xmm2996; SSE42-NEXT: pand %xmm10, %xmm2997; SSE42-NEXT: movdqa %xmm9, %xmm3998; SSE42-NEXT: pshufb %xmm2, %xmm3999; SSE42-NEXT: paddb %xmm8, %xmm31000; SSE42-NEXT: paddb %xmm1, %xmm31001; SSE42-NEXT: movdqa %xmm5, %xmm11002; SSE42-NEXT: pand %xmm10, %xmm11003; SSE42-NEXT: movdqa %xmm9, %xmm21004; SSE42-NEXT: pshufb %xmm1, %xmm21005; SSE42-NEXT: psrlw $4, %xmm51006; SSE42-NEXT: pand %xmm10, %xmm51007; SSE42-NEXT: movdqa %xmm9, %xmm81008; SSE42-NEXT: pshufb %xmm5, %xmm81009; SSE42-NEXT: paddb %xmm2, %xmm81010; SSE42-NEXT: movdqa %xmm4, %xmm11011; SSE42-NEXT: pand %xmm10, %xmm11012; SSE42-NEXT: movdqa %xmm9, %xmm21013; SSE42-NEXT: pshufb %xmm1, %xmm21014; SSE42-NEXT: psrlw $4, %xmm41015; SSE42-NEXT: pand %xmm10, %xmm41016; SSE42-NEXT: movdqa %xmm9, %xmm11017; SSE42-NEXT: pshufb %xmm4, %xmm11018; SSE42-NEXT: paddb %xmm2, %xmm11019; SSE42-NEXT: paddb %xmm8, %xmm11020; SSE42-NEXT: movdqa %xmm7, %xmm21021; SSE42-NEXT: pand %xmm10, %xmm21022; SSE42-NEXT: movdqa %xmm9, %xmm41023; SSE42-NEXT: pshufb %xmm2, %xmm41024; SSE42-NEXT: psrlw $4, %xmm71025; SSE42-NEXT: pand %xmm10, %xmm71026; SSE42-NEXT: movdqa %xmm9, %xmm21027; SSE42-NEXT: pshufb %xmm7, %xmm21028; SSE42-NEXT: paddb %xmm4, %xmm21029; SSE42-NEXT: movdqa %xmm6, %xmm41030; SSE42-NEXT: pand %xmm10, %xmm41031; SSE42-NEXT: movdqa %xmm9, %xmm51032; SSE42-NEXT: pshufb %xmm4, %xmm51033; SSE42-NEXT: psrlw $4, %xmm61034; SSE42-NEXT: pand %xmm10, %xmm61035; SSE42-NEXT: pshufb %xmm6, %xmm91036; SSE42-NEXT: paddb %xmm5, %xmm91037; SSE42-NEXT: paddb %xmm2, %xmm91038; SSE42-NEXT: pxor %xmm2, %xmm21039; SSE42-NEXT: psadbw %xmm2, %xmm01040; SSE42-NEXT: psadbw %xmm2, %xmm31041; SSE42-NEXT: psadbw %xmm2, %xmm11042; SSE42-NEXT: psadbw %xmm2, %xmm91043; SSE42-NEXT: movdqa %xmm0, %xmm21044; SSE42-NEXT: punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm3[1]1045; SSE42-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]1046; SSE42-NEXT: paddq %xmm2, %xmm01047; SSE42-NEXT: movdqa %xmm1, %xmm21048; SSE42-NEXT: punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm9[1]1049; SSE42-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm9[0]1050; SSE42-NEXT: paddq %xmm2, %xmm11051; SSE42-NEXT: retq1052;1053; AVX2-LABEL: reduce_ctpop_v4i64_buildvector_v4i64:1054; AVX2: # %bb.0:1055; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]1056; AVX2-NEXT: vpand %ymm4, %ymm0, %ymm51057; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]1058; AVX2-NEXT: # ymm6 = mem[0,1,0,1]1059; AVX2-NEXT: vpshufb %ymm5, %ymm6, %ymm51060; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm01061; AVX2-NEXT: vpand %ymm4, %ymm0, %ymm01062; AVX2-NEXT: vpshufb %ymm0, %ymm6, %ymm01063; AVX2-NEXT: vpaddb %ymm5, %ymm0, %ymm01064; AVX2-NEXT: vpxor %xmm5, %xmm5, %xmm51065; AVX2-NEXT: vpsadbw %ymm5, %ymm0, %ymm01066; AVX2-NEXT: vpand %ymm4, %ymm1, %ymm71067; AVX2-NEXT: vpshufb %ymm7, %ymm6, %ymm71068; AVX2-NEXT: vpsrlw $4, %ymm1, %ymm11069; AVX2-NEXT: vpand %ymm4, %ymm1, %ymm11070; AVX2-NEXT: vpshufb %ymm1, %ymm6, %ymm11071; AVX2-NEXT: vpaddb %ymm7, %ymm1, %ymm11072; AVX2-NEXT: vpsadbw %ymm5, %ymm1, %ymm11073; AVX2-NEXT: vpand %ymm4, %ymm2, %ymm71074; AVX2-NEXT: vpshufb %ymm7, %ymm6, %ymm71075; AVX2-NEXT: vpsrlw $4, %ymm2, %ymm21076; AVX2-NEXT: vpand %ymm4, %ymm2, %ymm21077; AVX2-NEXT: vpshufb %ymm2, %ymm6, %ymm21078; AVX2-NEXT: vpaddb %ymm7, %ymm2, %ymm21079; AVX2-NEXT: vpsadbw %ymm5, %ymm2, %ymm21080; AVX2-NEXT: vpand %ymm4, %ymm3, %ymm71081; AVX2-NEXT: vpshufb %ymm7, %ymm6, %ymm71082; AVX2-NEXT: vpsrlw $4, %ymm3, %ymm31083; AVX2-NEXT: vpand %ymm4, %ymm3, %ymm31084; AVX2-NEXT: vpshufb %ymm3, %ymm6, %ymm31085; AVX2-NEXT: vpaddb %ymm7, %ymm3, %ymm31086; AVX2-NEXT: vpsadbw %ymm5, %ymm3, %ymm31087; AVX2-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm1[2,3],ymm3[2,3]1088; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm11089; AVX2-NEXT: vpaddq %ymm4, %ymm1, %ymm11090; AVX2-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm0[2,3],ymm2[2,3]1091; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm01092; AVX2-NEXT: vpaddq %ymm3, %ymm0, %ymm01093; AVX2-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]1094; AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]1095; AVX2-NEXT: vpaddq %ymm2, %ymm0, %ymm01096; AVX2-NEXT: retq1097;1098; AVX512VL-LABEL: reduce_ctpop_v4i64_buildvector_v4i64:1099; AVX512VL: # %bb.0:1100; AVX512VL-NEXT: vpbroadcastb {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]1101; AVX512VL-NEXT: vpand %ymm4, %ymm0, %ymm51102; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]1103; AVX512VL-NEXT: # ymm6 = mem[0,1,0,1]1104; AVX512VL-NEXT: vpshufb %ymm5, %ymm6, %ymm51105; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm01106; AVX512VL-NEXT: vpand %ymm4, %ymm0, %ymm01107; AVX512VL-NEXT: vpshufb %ymm0, %ymm6, %ymm01108; AVX512VL-NEXT: vpaddb %ymm5, %ymm0, %ymm01109; AVX512VL-NEXT: vpxor %xmm5, %xmm5, %xmm51110; AVX512VL-NEXT: vpsadbw %ymm5, %ymm0, %ymm01111; AVX512VL-NEXT: vpand %ymm4, %ymm1, %ymm71112; AVX512VL-NEXT: vpshufb %ymm7, %ymm6, %ymm71113; AVX512VL-NEXT: vpsrlw $4, %ymm1, %ymm11114; AVX512VL-NEXT: vpand %ymm4, %ymm1, %ymm11115; AVX512VL-NEXT: vpshufb %ymm1, %ymm6, %ymm11116; AVX512VL-NEXT: vpaddb %ymm7, %ymm1, %ymm11117; AVX512VL-NEXT: vpsadbw %ymm5, %ymm1, %ymm11118; AVX512VL-NEXT: vpand %ymm4, %ymm2, %ymm71119; AVX512VL-NEXT: vpshufb %ymm7, %ymm6, %ymm71120; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm21121; AVX512VL-NEXT: vpand %ymm4, %ymm2, %ymm21122; AVX512VL-NEXT: vpshufb %ymm2, %ymm6, %ymm21123; AVX512VL-NEXT: vpaddb %ymm7, %ymm2, %ymm21124; AVX512VL-NEXT: vpsadbw %ymm5, %ymm2, %ymm21125; AVX512VL-NEXT: vpand %ymm4, %ymm3, %ymm71126; AVX512VL-NEXT: vpshufb %ymm7, %ymm6, %ymm71127; AVX512VL-NEXT: vpsrlw $4, %ymm3, %ymm31128; AVX512VL-NEXT: vpand %ymm4, %ymm3, %ymm31129; AVX512VL-NEXT: vpshufb %ymm3, %ymm6, %ymm31130; AVX512VL-NEXT: vpaddb %ymm7, %ymm3, %ymm31131; AVX512VL-NEXT: vpsadbw %ymm5, %ymm3, %ymm31132; AVX512VL-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm1[2,3],ymm3[2,3]1133; AVX512VL-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm11134; AVX512VL-NEXT: vpaddq %ymm4, %ymm1, %ymm11135; AVX512VL-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm0[2,3],ymm2[2,3]1136; AVX512VL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm01137; AVX512VL-NEXT: vpaddq %ymm3, %ymm0, %ymm01138; AVX512VL-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]1139; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]1140; AVX512VL-NEXT: vpaddq %ymm2, %ymm0, %ymm01141; AVX512VL-NEXT: retq1142;1143; AVX512VPOPCNT-LABEL: reduce_ctpop_v4i64_buildvector_v4i64:1144; AVX512VPOPCNT: # %bb.0:1145; AVX512VPOPCNT-NEXT: vpopcntq %ymm0, %ymm01146; AVX512VPOPCNT-NEXT: vpopcntq %ymm1, %ymm11147; AVX512VPOPCNT-NEXT: vpopcntq %ymm2, %ymm21148; AVX512VPOPCNT-NEXT: vpopcntq %ymm3, %ymm31149; AVX512VPOPCNT-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm1[2,3],ymm3[2,3]1150; AVX512VPOPCNT-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm11151; AVX512VPOPCNT-NEXT: vpaddq %ymm4, %ymm1, %ymm11152; AVX512VPOPCNT-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm0[2,3],ymm2[2,3]1153; AVX512VPOPCNT-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm01154; AVX512VPOPCNT-NEXT: vpaddq %ymm3, %ymm0, %ymm01155; AVX512VPOPCNT-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]1156; AVX512VPOPCNT-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]1157; AVX512VPOPCNT-NEXT: vpaddq %ymm2, %ymm0, %ymm01158; AVX512VPOPCNT-NEXT: retq1159 %p0 = tail call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a0)1160 %p1 = tail call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a1)1161 %p2 = tail call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a2)1162 %p3 = tail call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a3)1163 %r0 = tail call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %p0)1164 %r1 = tail call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %p1)1165 %r2 = tail call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %p2)1166 %r3 = tail call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %p3)1167 %v0 = insertelement <4 x i64> undef, i64 %r0, i64 01168 %v1 = insertelement <4 x i64> %v0, i64 %r1, i64 11169 %v2 = insertelement <4 x i64> %v1, i64 %r2, i64 21170 %v3 = insertelement <4 x i64> %v2, i64 %r3, i64 31171 ret <4 x i64> %v31172}1173 1174define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> %a2, <4 x i64> %a3, <4 x i64> %a4, <4 x i64> %a5, <4 x i64> %a6, <4 x i64> %a7) nounwind {1175; SSE42-LABEL: reduce_ctpop_v4i64_buildvector_v8i32:1176; SSE42: # %bb.0:1177; SSE42-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm91178; SSE42-NEXT: movdqa {{.*#+}} xmm10 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]1179; SSE42-NEXT: movdqa %xmm1, %xmm111180; SSE42-NEXT: pand %xmm10, %xmm111181; SSE42-NEXT: movdqa {{.*#+}} xmm8 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]1182; SSE42-NEXT: movdqa %xmm8, %xmm121183; SSE42-NEXT: pshufb %xmm11, %xmm121184; SSE42-NEXT: psrlw $4, %xmm11185; SSE42-NEXT: pand %xmm10, %xmm11186; SSE42-NEXT: movdqa %xmm8, %xmm111187; SSE42-NEXT: pshufb %xmm1, %xmm111188; SSE42-NEXT: paddb %xmm12, %xmm111189; SSE42-NEXT: movdqa %xmm0, %xmm11190; SSE42-NEXT: pand %xmm10, %xmm11191; SSE42-NEXT: movdqa %xmm8, %xmm121192; SSE42-NEXT: pshufb %xmm1, %xmm121193; SSE42-NEXT: psrlw $4, %xmm01194; SSE42-NEXT: pand %xmm10, %xmm01195; SSE42-NEXT: movdqa %xmm8, %xmm11196; SSE42-NEXT: pshufb %xmm0, %xmm11197; SSE42-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm01198; SSE42-NEXT: paddb %xmm12, %xmm11199; SSE42-NEXT: paddb %xmm11, %xmm11200; SSE42-NEXT: movdqa %xmm3, %xmm111201; SSE42-NEXT: pand %xmm10, %xmm111202; SSE42-NEXT: movdqa %xmm8, %xmm121203; SSE42-NEXT: pshufb %xmm11, %xmm121204; SSE42-NEXT: psrlw $4, %xmm31205; SSE42-NEXT: pand %xmm10, %xmm31206; SSE42-NEXT: movdqa %xmm8, %xmm131207; SSE42-NEXT: pshufb %xmm3, %xmm131208; SSE42-NEXT: paddb %xmm12, %xmm131209; SSE42-NEXT: movdqa %xmm2, %xmm31210; SSE42-NEXT: pand %xmm10, %xmm31211; SSE42-NEXT: movdqa %xmm8, %xmm121212; SSE42-NEXT: pshufb %xmm3, %xmm121213; SSE42-NEXT: psrlw $4, %xmm21214; SSE42-NEXT: pand %xmm10, %xmm21215; SSE42-NEXT: movdqa %xmm8, %xmm31216; SSE42-NEXT: pshufb %xmm2, %xmm31217; SSE42-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm111218; SSE42-NEXT: paddb %xmm12, %xmm31219; SSE42-NEXT: paddb %xmm13, %xmm31220; SSE42-NEXT: movdqa %xmm5, %xmm21221; SSE42-NEXT: pand %xmm10, %xmm21222; SSE42-NEXT: movdqa %xmm8, %xmm121223; SSE42-NEXT: pshufb %xmm2, %xmm121224; SSE42-NEXT: psrlw $4, %xmm51225; SSE42-NEXT: pand %xmm10, %xmm51226; SSE42-NEXT: movdqa %xmm8, %xmm131227; SSE42-NEXT: pshufb %xmm5, %xmm131228; SSE42-NEXT: paddb %xmm12, %xmm131229; SSE42-NEXT: movdqa %xmm4, %xmm21230; SSE42-NEXT: pand %xmm10, %xmm21231; SSE42-NEXT: movdqa %xmm8, %xmm51232; SSE42-NEXT: pshufb %xmm2, %xmm51233; SSE42-NEXT: psrlw $4, %xmm41234; SSE42-NEXT: pand %xmm10, %xmm41235; SSE42-NEXT: movdqa %xmm8, %xmm21236; SSE42-NEXT: pshufb %xmm4, %xmm21237; SSE42-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm121238; SSE42-NEXT: paddb %xmm5, %xmm21239; SSE42-NEXT: paddb %xmm13, %xmm21240; SSE42-NEXT: movdqa %xmm7, %xmm41241; SSE42-NEXT: pand %xmm10, %xmm41242; SSE42-NEXT: movdqa %xmm8, %xmm51243; SSE42-NEXT: pshufb %xmm4, %xmm51244; SSE42-NEXT: psrlw $4, %xmm71245; SSE42-NEXT: pand %xmm10, %xmm71246; SSE42-NEXT: movdqa %xmm8, %xmm131247; SSE42-NEXT: pshufb %xmm7, %xmm131248; SSE42-NEXT: paddb %xmm5, %xmm131249; SSE42-NEXT: movdqa %xmm6, %xmm41250; SSE42-NEXT: pand %xmm10, %xmm41251; SSE42-NEXT: movdqa %xmm8, %xmm51252; SSE42-NEXT: pshufb %xmm4, %xmm51253; SSE42-NEXT: psrlw $4, %xmm61254; SSE42-NEXT: pand %xmm10, %xmm61255; SSE42-NEXT: movdqa %xmm8, %xmm41256; SSE42-NEXT: pshufb %xmm6, %xmm41257; SSE42-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm61258; SSE42-NEXT: paddb %xmm5, %xmm41259; SSE42-NEXT: paddb %xmm13, %xmm41260; SSE42-NEXT: movdqa %xmm6, %xmm51261; SSE42-NEXT: pand %xmm10, %xmm51262; SSE42-NEXT: movdqa %xmm8, %xmm71263; SSE42-NEXT: pshufb %xmm5, %xmm71264; SSE42-NEXT: psrlw $4, %xmm61265; SSE42-NEXT: pand %xmm10, %xmm61266; SSE42-NEXT: movdqa %xmm8, %xmm131267; SSE42-NEXT: pshufb %xmm6, %xmm131268; SSE42-NEXT: paddb %xmm7, %xmm131269; SSE42-NEXT: movdqa %xmm12, %xmm51270; SSE42-NEXT: pand %xmm10, %xmm51271; SSE42-NEXT: movdqa %xmm8, %xmm61272; SSE42-NEXT: pshufb %xmm5, %xmm61273; SSE42-NEXT: psrlw $4, %xmm121274; SSE42-NEXT: pand %xmm10, %xmm121275; SSE42-NEXT: movdqa %xmm8, %xmm51276; SSE42-NEXT: pshufb %xmm12, %xmm51277; SSE42-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm71278; SSE42-NEXT: paddb %xmm6, %xmm51279; SSE42-NEXT: paddb %xmm13, %xmm51280; SSE42-NEXT: movdqa %xmm7, %xmm61281; SSE42-NEXT: pand %xmm10, %xmm61282; SSE42-NEXT: movdqa %xmm8, %xmm121283; SSE42-NEXT: pshufb %xmm6, %xmm121284; SSE42-NEXT: psrlw $4, %xmm71285; SSE42-NEXT: pand %xmm10, %xmm71286; SSE42-NEXT: movdqa %xmm8, %xmm131287; SSE42-NEXT: pshufb %xmm7, %xmm131288; SSE42-NEXT: paddb %xmm12, %xmm131289; SSE42-NEXT: movdqa %xmm11, %xmm61290; SSE42-NEXT: pand %xmm10, %xmm61291; SSE42-NEXT: movdqa %xmm8, %xmm71292; SSE42-NEXT: pshufb %xmm6, %xmm71293; SSE42-NEXT: psrlw $4, %xmm111294; SSE42-NEXT: pand %xmm10, %xmm111295; SSE42-NEXT: movdqa %xmm8, %xmm61296; SSE42-NEXT: pshufb %xmm11, %xmm61297; SSE42-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm111298; SSE42-NEXT: paddb %xmm7, %xmm61299; SSE42-NEXT: paddb %xmm13, %xmm61300; SSE42-NEXT: movdqa %xmm11, %xmm71301; SSE42-NEXT: pand %xmm10, %xmm71302; SSE42-NEXT: movdqa %xmm8, %xmm121303; SSE42-NEXT: pshufb %xmm7, %xmm121304; SSE42-NEXT: psrlw $4, %xmm111305; SSE42-NEXT: pand %xmm10, %xmm111306; SSE42-NEXT: movdqa %xmm8, %xmm131307; SSE42-NEXT: pshufb %xmm11, %xmm131308; SSE42-NEXT: paddb %xmm12, %xmm131309; SSE42-NEXT: movdqa %xmm0, %xmm71310; SSE42-NEXT: pand %xmm10, %xmm71311; SSE42-NEXT: movdqa %xmm8, %xmm111312; SSE42-NEXT: pshufb %xmm7, %xmm111313; SSE42-NEXT: psrlw $4, %xmm01314; SSE42-NEXT: pand %xmm10, %xmm01315; SSE42-NEXT: movdqa %xmm8, %xmm71316; SSE42-NEXT: pshufb %xmm0, %xmm71317; SSE42-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm01318; SSE42-NEXT: paddb %xmm11, %xmm71319; SSE42-NEXT: paddb %xmm13, %xmm71320; SSE42-NEXT: movdqa %xmm0, %xmm111321; SSE42-NEXT: pand %xmm10, %xmm111322; SSE42-NEXT: movdqa %xmm8, %xmm121323; SSE42-NEXT: pshufb %xmm11, %xmm121324; SSE42-NEXT: psrlw $4, %xmm01325; SSE42-NEXT: pand %xmm10, %xmm01326; SSE42-NEXT: movdqa %xmm8, %xmm111327; SSE42-NEXT: pshufb %xmm0, %xmm111328; SSE42-NEXT: paddb %xmm12, %xmm111329; SSE42-NEXT: movdqa %xmm9, %xmm01330; SSE42-NEXT: pand %xmm10, %xmm01331; SSE42-NEXT: movdqa %xmm8, %xmm121332; SSE42-NEXT: pshufb %xmm0, %xmm121333; SSE42-NEXT: psrlw $4, %xmm91334; SSE42-NEXT: pand %xmm10, %xmm91335; SSE42-NEXT: pshufb %xmm9, %xmm81336; SSE42-NEXT: paddb %xmm12, %xmm81337; SSE42-NEXT: paddb %xmm11, %xmm81338; SSE42-NEXT: pxor %xmm9, %xmm91339; SSE42-NEXT: psadbw %xmm9, %xmm11340; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]1341; SSE42-NEXT: paddq %xmm1, %xmm01342; SSE42-NEXT: psadbw %xmm9, %xmm31343; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]1344; SSE42-NEXT: paddq %xmm3, %xmm11345; SSE42-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]1346; SSE42-NEXT: psadbw %xmm9, %xmm21347; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]1348; SSE42-NEXT: paddq %xmm2, %xmm11349; SSE42-NEXT: psadbw %xmm9, %xmm41350; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]1351; SSE42-NEXT: paddq %xmm4, %xmm21352; SSE42-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]1353; SSE42-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]1354; SSE42-NEXT: psadbw %xmm9, %xmm51355; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm5[2,3,2,3]1356; SSE42-NEXT: paddq %xmm5, %xmm11357; SSE42-NEXT: psadbw %xmm9, %xmm61358; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm6[2,3,2,3]1359; SSE42-NEXT: paddq %xmm6, %xmm21360; SSE42-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]1361; SSE42-NEXT: psadbw %xmm9, %xmm71362; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm7[2,3,2,3]1363; SSE42-NEXT: paddq %xmm7, %xmm21364; SSE42-NEXT: psadbw %xmm9, %xmm81365; SSE42-NEXT: pshufd {{.*#+}} xmm3 = xmm8[2,3,2,3]1366; SSE42-NEXT: paddq %xmm8, %xmm31367; SSE42-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]1368; SSE42-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]1369; SSE42-NEXT: retq1370;1371; AVX2-LABEL: reduce_ctpop_v4i64_buildvector_v8i32:1372; AVX2: # %bb.0:1373; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm8 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]1374; AVX2-NEXT: vpand %ymm0, %ymm8, %ymm101375; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm9 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]1376; AVX2-NEXT: # ymm9 = mem[0,1,0,1]1377; AVX2-NEXT: vpshufb %ymm10, %ymm9, %ymm101378; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm01379; AVX2-NEXT: vpand %ymm0, %ymm8, %ymm01380; AVX2-NEXT: vpshufb %ymm0, %ymm9, %ymm01381; AVX2-NEXT: vpaddb %ymm0, %ymm10, %ymm01382; AVX2-NEXT: vpxor %xmm10, %xmm10, %xmm101383; AVX2-NEXT: vpsadbw %ymm0, %ymm10, %ymm01384; AVX2-NEXT: vpand %ymm1, %ymm8, %ymm111385; AVX2-NEXT: vpshufb %ymm11, %ymm9, %ymm111386; AVX2-NEXT: vpsrlw $4, %ymm1, %ymm11387; AVX2-NEXT: vpand %ymm1, %ymm8, %ymm11388; AVX2-NEXT: vpshufb %ymm1, %ymm9, %ymm11389; AVX2-NEXT: vpaddb %ymm1, %ymm11, %ymm11390; AVX2-NEXT: vpsadbw %ymm1, %ymm10, %ymm11391; AVX2-NEXT: vpand %ymm2, %ymm8, %ymm111392; AVX2-NEXT: vpshufb %ymm11, %ymm9, %ymm111393; AVX2-NEXT: vpsrlw $4, %ymm2, %ymm21394; AVX2-NEXT: vpand %ymm2, %ymm8, %ymm21395; AVX2-NEXT: vpshufb %ymm2, %ymm9, %ymm21396; AVX2-NEXT: vpaddb %ymm2, %ymm11, %ymm21397; AVX2-NEXT: vpsadbw %ymm2, %ymm10, %ymm21398; AVX2-NEXT: vpand %ymm3, %ymm8, %ymm111399; AVX2-NEXT: vpshufb %ymm11, %ymm9, %ymm111400; AVX2-NEXT: vpsrlw $4, %ymm3, %ymm31401; AVX2-NEXT: vpand %ymm3, %ymm8, %ymm31402; AVX2-NEXT: vpshufb %ymm3, %ymm9, %ymm31403; AVX2-NEXT: vpaddb %ymm3, %ymm11, %ymm31404; AVX2-NEXT: vpsadbw %ymm3, %ymm10, %ymm31405; AVX2-NEXT: vpand %ymm4, %ymm8, %ymm111406; AVX2-NEXT: vpshufb %ymm11, %ymm9, %ymm111407; AVX2-NEXT: vpsrlw $4, %ymm4, %ymm41408; AVX2-NEXT: vpand %ymm4, %ymm8, %ymm41409; AVX2-NEXT: vpshufb %ymm4, %ymm9, %ymm41410; AVX2-NEXT: vpaddb %ymm4, %ymm11, %ymm41411; AVX2-NEXT: vpsadbw %ymm4, %ymm10, %ymm41412; AVX2-NEXT: vpand %ymm5, %ymm8, %ymm111413; AVX2-NEXT: vpshufb %ymm11, %ymm9, %ymm111414; AVX2-NEXT: vpsrlw $4, %ymm5, %ymm51415; AVX2-NEXT: vpand %ymm5, %ymm8, %ymm51416; AVX2-NEXT: vpshufb %ymm5, %ymm9, %ymm51417; AVX2-NEXT: vpaddb %ymm5, %ymm11, %ymm51418; AVX2-NEXT: vpsadbw %ymm5, %ymm10, %ymm51419; AVX2-NEXT: vpand %ymm6, %ymm8, %ymm111420; AVX2-NEXT: vpshufb %ymm11, %ymm9, %ymm111421; AVX2-NEXT: vpsrlw $4, %ymm6, %ymm61422; AVX2-NEXT: vpand %ymm6, %ymm8, %ymm61423; AVX2-NEXT: vpshufb %ymm6, %ymm9, %ymm61424; AVX2-NEXT: vpaddb %ymm6, %ymm11, %ymm61425; AVX2-NEXT: vpsadbw %ymm6, %ymm10, %ymm61426; AVX2-NEXT: vpand %ymm7, %ymm8, %ymm111427; AVX2-NEXT: vpshufb %ymm11, %ymm9, %ymm111428; AVX2-NEXT: vpsrlw $4, %ymm7, %ymm71429; AVX2-NEXT: vpand %ymm7, %ymm8, %ymm71430; AVX2-NEXT: vpshufb %ymm7, %ymm9, %ymm71431; AVX2-NEXT: vpaddb %ymm7, %ymm11, %ymm71432; AVX2-NEXT: vpsadbw %ymm7, %ymm10, %ymm71433; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm81434; AVX2-NEXT: vpaddq %xmm0, %xmm8, %xmm01435; AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[2,3,2,3]1436; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm91437; AVX2-NEXT: vpaddq %xmm1, %xmm9, %xmm11438; AVX2-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[2,3,2,3]1439; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm101440; AVX2-NEXT: vpaddq %xmm2, %xmm10, %xmm21441; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm2[2,3,2,3]1442; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm111443; AVX2-NEXT: vpaddq %xmm3, %xmm11, %xmm31444; AVX2-NEXT: vpshufd {{.*#+}} xmm11 = xmm3[2,3,2,3]1445; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm121446; AVX2-NEXT: vpaddq %xmm4, %xmm12, %xmm41447; AVX2-NEXT: vpshufd {{.*#+}} xmm12 = xmm4[2,3,2,3]1448; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm131449; AVX2-NEXT: vpaddq %xmm5, %xmm13, %xmm51450; AVX2-NEXT: vpshufd {{.*#+}} xmm13 = xmm5[2,3,2,3]1451; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm141452; AVX2-NEXT: vpaddq %xmm6, %xmm14, %xmm61453; AVX2-NEXT: vpshufd {{.*#+}} xmm14 = xmm6[2,3,2,3]1454; AVX2-NEXT: vextracti128 $1, %ymm7, %xmm151455; AVX2-NEXT: vpaddq %xmm7, %xmm15, %xmm71456; AVX2-NEXT: vpshufd {{.*#+}} xmm15 = xmm7[2,3,2,3]1457; AVX2-NEXT: vpaddq %xmm0, %xmm8, %xmm01458; AVX2-NEXT: vpaddq %xmm1, %xmm9, %xmm11459; AVX2-NEXT: vmovd %xmm1, %eax1460; AVX2-NEXT: vpaddq %xmm2, %xmm10, %xmm11461; AVX2-NEXT: vmovd %xmm1, %ecx1462; AVX2-NEXT: vpaddq %xmm3, %xmm11, %xmm11463; AVX2-NEXT: vmovd %xmm1, %edx1464; AVX2-NEXT: vpaddq %xmm4, %xmm12, %xmm11465; AVX2-NEXT: vpaddq %xmm5, %xmm13, %xmm21466; AVX2-NEXT: vmovd %xmm2, %esi1467; AVX2-NEXT: vpaddq %xmm6, %xmm14, %xmm21468; AVX2-NEXT: vmovd %xmm2, %edi1469; AVX2-NEXT: vpaddq %xmm7, %xmm15, %xmm21470; AVX2-NEXT: vmovd %xmm2, %r8d1471; AVX2-NEXT: vpinsrd $1, %esi, %xmm1, %xmm11472; AVX2-NEXT: vpinsrd $2, %edi, %xmm1, %xmm11473; AVX2-NEXT: vpinsrd $3, %r8d, %xmm1, %xmm11474; AVX2-NEXT: vpinsrd $1, %eax, %xmm0, %xmm01475; AVX2-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm01476; AVX2-NEXT: vpinsrd $3, %edx, %xmm0, %xmm01477; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm01478; AVX2-NEXT: retq1479;1480; AVX512VL-LABEL: reduce_ctpop_v4i64_buildvector_v8i32:1481; AVX512VL: # %bb.0:1482; AVX512VL-NEXT: vpbroadcastb {{.*#+}} ymm8 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]1483; AVX512VL-NEXT: vpand %ymm0, %ymm8, %ymm91484; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm10 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]1485; AVX512VL-NEXT: # ymm10 = mem[0,1,0,1]1486; AVX512VL-NEXT: vpshufb %ymm9, %ymm10, %ymm91487; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm01488; AVX512VL-NEXT: vpand %ymm0, %ymm8, %ymm01489; AVX512VL-NEXT: vpshufb %ymm0, %ymm10, %ymm01490; AVX512VL-NEXT: vpaddb %ymm0, %ymm9, %ymm01491; AVX512VL-NEXT: vpxor %xmm9, %xmm9, %xmm91492; AVX512VL-NEXT: vpsadbw %ymm0, %ymm9, %ymm01493; AVX512VL-NEXT: vpand %ymm1, %ymm8, %ymm111494; AVX512VL-NEXT: vpshufb %ymm11, %ymm10, %ymm111495; AVX512VL-NEXT: vpsrlw $4, %ymm1, %ymm11496; AVX512VL-NEXT: vpand %ymm1, %ymm8, %ymm11497; AVX512VL-NEXT: vpshufb %ymm1, %ymm10, %ymm11498; AVX512VL-NEXT: vpaddb %ymm1, %ymm11, %ymm11499; AVX512VL-NEXT: vpsadbw %ymm1, %ymm9, %ymm11500; AVX512VL-NEXT: vpand %ymm2, %ymm8, %ymm111501; AVX512VL-NEXT: vpshufb %ymm11, %ymm10, %ymm111502; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm21503; AVX512VL-NEXT: vpand %ymm2, %ymm8, %ymm21504; AVX512VL-NEXT: vpshufb %ymm2, %ymm10, %ymm21505; AVX512VL-NEXT: vpaddb %ymm2, %ymm11, %ymm21506; AVX512VL-NEXT: vpsadbw %ymm2, %ymm9, %ymm21507; AVX512VL-NEXT: vpand %ymm3, %ymm8, %ymm111508; AVX512VL-NEXT: vpshufb %ymm11, %ymm10, %ymm111509; AVX512VL-NEXT: vpsrlw $4, %ymm3, %ymm31510; AVX512VL-NEXT: vpand %ymm3, %ymm8, %ymm31511; AVX512VL-NEXT: vpshufb %ymm3, %ymm10, %ymm31512; AVX512VL-NEXT: vpaddb %ymm3, %ymm11, %ymm31513; AVX512VL-NEXT: vpsadbw %ymm3, %ymm9, %ymm31514; AVX512VL-NEXT: vpand %ymm4, %ymm8, %ymm111515; AVX512VL-NEXT: vpshufb %ymm11, %ymm10, %ymm111516; AVX512VL-NEXT: vpsrlw $4, %ymm4, %ymm41517; AVX512VL-NEXT: vpand %ymm4, %ymm8, %ymm41518; AVX512VL-NEXT: vpshufb %ymm4, %ymm10, %ymm41519; AVX512VL-NEXT: vpaddb %ymm4, %ymm11, %ymm41520; AVX512VL-NEXT: vpsadbw %ymm4, %ymm9, %ymm41521; AVX512VL-NEXT: vpand %ymm5, %ymm8, %ymm111522; AVX512VL-NEXT: vpshufb %ymm11, %ymm10, %ymm111523; AVX512VL-NEXT: vpsrlw $4, %ymm5, %ymm51524; AVX512VL-NEXT: vpand %ymm5, %ymm8, %ymm51525; AVX512VL-NEXT: vpshufb %ymm5, %ymm10, %ymm51526; AVX512VL-NEXT: vpaddb %ymm5, %ymm11, %ymm51527; AVX512VL-NEXT: vpsadbw %ymm5, %ymm9, %ymm51528; AVX512VL-NEXT: vpand %ymm6, %ymm8, %ymm111529; AVX512VL-NEXT: vpshufb %ymm11, %ymm10, %ymm111530; AVX512VL-NEXT: vpsrlw $4, %ymm6, %ymm61531; AVX512VL-NEXT: vpand %ymm6, %ymm8, %ymm61532; AVX512VL-NEXT: vpshufb %ymm6, %ymm10, %ymm61533; AVX512VL-NEXT: vpaddb %ymm6, %ymm11, %ymm61534; AVX512VL-NEXT: vpsadbw %ymm6, %ymm9, %ymm61535; AVX512VL-NEXT: vpand %ymm7, %ymm8, %ymm111536; AVX512VL-NEXT: vpshufb %ymm11, %ymm10, %ymm111537; AVX512VL-NEXT: vpsrlw $4, %ymm7, %ymm71538; AVX512VL-NEXT: vpand %ymm7, %ymm8, %ymm71539; AVX512VL-NEXT: vpshufb %ymm7, %ymm10, %ymm71540; AVX512VL-NEXT: vpaddb %ymm7, %ymm11, %ymm71541; AVX512VL-NEXT: vpsadbw %ymm7, %ymm9, %ymm71542; AVX512VL-NEXT: vpmovqb %ymm0, %xmm01543; AVX512VL-NEXT: vpmovqb %ymm1, %xmm11544; AVX512VL-NEXT: vpmovqb %ymm2, %xmm21545; AVX512VL-NEXT: vpmovqb %ymm3, %xmm31546; AVX512VL-NEXT: vpmovqb %ymm4, %xmm41547; AVX512VL-NEXT: vpmovqb %ymm5, %xmm51548; AVX512VL-NEXT: vpmovqb %ymm6, %xmm61549; AVX512VL-NEXT: vpmovqb %ymm7, %xmm71550; AVX512VL-NEXT: vinserti128 $1, %xmm5, %ymm4, %ymm41551; AVX512VL-NEXT: vinserti128 $1, %xmm7, %ymm6, %ymm51552; AVX512VL-NEXT: vinserti64x4 $1, %ymm4, %zmm5, %zmm41553; AVX512VL-NEXT: vpxor %xmm5, %xmm5, %xmm51554; AVX512VL-NEXT: vpsadbw %zmm5, %zmm4, %zmm41555; AVX512VL-NEXT: vpmovsxbd {{.*#+}} ymm6 = [8,12,8,12,8,12,0,4]1556; AVX512VL-NEXT: vpermd %zmm4, %zmm6, %zmm41557; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm01558; AVX512VL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm11559; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm01560; AVX512VL-NEXT: vpsadbw %zmm5, %zmm0, %zmm01561; AVX512VL-NEXT: vpmovsxbd {{.*#+}} xmm1 = [8,12,0,4]1562; AVX512VL-NEXT: vpermd %zmm0, %zmm1, %zmm01563; AVX512VL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]1564; AVX512VL-NEXT: retq1565;1566; AVX512VPOPCNT-LABEL: reduce_ctpop_v4i64_buildvector_v8i32:1567; AVX512VPOPCNT: # %bb.0:1568; AVX512VPOPCNT-NEXT: vpopcntq %ymm0, %ymm01569; AVX512VPOPCNT-NEXT: vpopcntq %ymm1, %ymm11570; AVX512VPOPCNT-NEXT: vpopcntq %ymm2, %ymm21571; AVX512VPOPCNT-NEXT: vpopcntq %ymm3, %ymm31572; AVX512VPOPCNT-NEXT: vpopcntq %ymm4, %ymm41573; AVX512VPOPCNT-NEXT: vpopcntq %ymm5, %ymm51574; AVX512VPOPCNT-NEXT: vpopcntq %ymm6, %ymm61575; AVX512VPOPCNT-NEXT: vpopcntq %ymm7, %ymm71576; AVX512VPOPCNT-NEXT: vpmovqb %ymm0, %xmm01577; AVX512VPOPCNT-NEXT: vpmovqb %ymm1, %xmm11578; AVX512VPOPCNT-NEXT: vpmovqb %ymm2, %xmm21579; AVX512VPOPCNT-NEXT: vpmovqb %ymm3, %xmm31580; AVX512VPOPCNT-NEXT: vpmovqb %ymm4, %xmm41581; AVX512VPOPCNT-NEXT: vpmovqb %ymm5, %xmm51582; AVX512VPOPCNT-NEXT: vpmovqb %ymm6, %xmm61583; AVX512VPOPCNT-NEXT: vpmovqb %ymm7, %xmm71584; AVX512VPOPCNT-NEXT: vinserti128 $1, %xmm5, %ymm4, %ymm41585; AVX512VPOPCNT-NEXT: vinserti128 $1, %xmm7, %ymm6, %ymm51586; AVX512VPOPCNT-NEXT: vinserti64x4 $1, %ymm4, %zmm5, %zmm41587; AVX512VPOPCNT-NEXT: vpxor %xmm5, %xmm5, %xmm51588; AVX512VPOPCNT-NEXT: vpsadbw %zmm5, %zmm4, %zmm41589; AVX512VPOPCNT-NEXT: vpmovsxbd {{.*#+}} ymm6 = [8,12,8,12,8,12,0,4]1590; AVX512VPOPCNT-NEXT: vpermd %zmm4, %zmm6, %zmm41591; AVX512VPOPCNT-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm01592; AVX512VPOPCNT-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm11593; AVX512VPOPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm01594; AVX512VPOPCNT-NEXT: vpsadbw %zmm5, %zmm0, %zmm01595; AVX512VPOPCNT-NEXT: vpmovsxbd {{.*#+}} xmm1 = [8,12,0,4]1596; AVX512VPOPCNT-NEXT: vpermd %zmm0, %zmm1, %zmm01597; AVX512VPOPCNT-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]1598; AVX512VPOPCNT-NEXT: retq1599 %p0 = tail call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a0)1600 %p1 = tail call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a1)1601 %p2 = tail call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a2)1602 %p3 = tail call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a3)1603 %p4 = tail call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a4)1604 %p5 = tail call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a5)1605 %p6 = tail call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a6)1606 %p7 = tail call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a7)1607 %r0 = tail call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %p0)1608 %r1 = tail call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %p1)1609 %r2 = tail call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %p2)1610 %r3 = tail call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %p3)1611 %r4 = tail call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %p4)1612 %r5 = tail call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %p5)1613 %r6 = tail call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %p6)1614 %r7 = tail call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %p7)1615 %t0 = trunc i64 %r0 to i321616 %t1 = trunc i64 %r1 to i321617 %t2 = trunc i64 %r2 to i321618 %t3 = trunc i64 %r3 to i321619 %t4 = trunc i64 %r4 to i321620 %t5 = trunc i64 %r5 to i321621 %t6 = trunc i64 %r6 to i321622 %t7 = trunc i64 %r7 to i321623 %v0 = insertelement <8 x i32> undef, i32 %t0, i64 01624 %v1 = insertelement <8 x i32> %v0, i32 %t1, i64 11625 %v2 = insertelement <8 x i32> %v1, i32 %t2, i64 21626 %v3 = insertelement <8 x i32> %v2, i32 %t3, i64 31627 %v4 = insertelement <8 x i32> %v3, i32 %t4, i64 41628 %v5 = insertelement <8 x i32> %v4, i32 %t5, i64 51629 %v6 = insertelement <8 x i32> %v5, i32 %t6, i64 61630 %v7 = insertelement <8 x i32> %v6, i32 %t7, i64 71631 ret <8 x i32> %v71632}1633 1634declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)1635declare <4 x i64> @llvm.ctpop.v4i64(<4 x i64>)1636declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)1637declare <8 x i32> @llvm.ctpop.v8i32(<8 x i32>)1638declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>)1639declare <16 x i8> @llvm.ctpop.v16i8(<16 x i8>)1640 1641declare i64 @llvm.vector.reduce.add.v2i64(<2 x i64>)1642declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64>)1643declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)1644declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)1645declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>)1646declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8>)1647