5037 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mtriple=x86_64 -mattr=+avx2 < %s | FileCheck %s --check-prefixes=CHECK,AVX23; RUN: llc -mtriple=x86_64 -mattr=+avx512f < %s | FileCheck %s --check-prefixes=CHECK,AVX512F4; RUN: llc -mtriple=x86_64 -mattr=+avx512f,+avx512vl,+avx512vbmi2 < %s | FileCheck %s --check-prefixes=CHECK,AVX512VL5 6define <4 x i32> @test_compress_v4i32(<4 x i32> %vec, <4 x i1> %mask, <4 x i32> %passthru) nounwind {7; AVX2-LABEL: test_compress_v4i32:8; AVX2: # %bb.0:9; AVX2-NEXT: vpslld $31, %xmm1, %xmm110; AVX2-NEXT: vpsrad $31, %xmm1, %xmm111; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)12; AVX2-NEXT: vpextrd $1, %xmm1, %eax13; AVX2-NEXT: vmovd %xmm1, %esi14; AVX2-NEXT: andl $1, %esi15; AVX2-NEXT: movl %esi, %edi16; AVX2-NEXT: subl %eax, %edi17; AVX2-NEXT: vpextrd $2, %xmm1, %edx18; AVX2-NEXT: subl %edx, %edi19; AVX2-NEXT: vpextrd $3, %xmm1, %ecx20; AVX2-NEXT: subl %ecx, %edi21; AVX2-NEXT: andl $3, %edi22; AVX2-NEXT: andl $1, %eax23; AVX2-NEXT: addq %rsi, %rax24; AVX2-NEXT: andl $1, %edx25; AVX2-NEXT: addq %rax, %rdx26; AVX2-NEXT: andl $1, %ecx27; AVX2-NEXT: addq %rdx, %rcx28; AVX2-NEXT: vextractps $3, %xmm0, %r8d29; AVX2-NEXT: cmpq $4, %rcx30; AVX2-NEXT: cmovbl -24(%rsp,%rdi,4), %r8d31; AVX2-NEXT: vmovss %xmm0, -{{[0-9]+}}(%rsp)32; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rsi,4)33; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rax,4)34; AVX2-NEXT: andl $3, %edx35; AVX2-NEXT: vextractps $3, %xmm0, -24(%rsp,%rdx,4)36; AVX2-NEXT: cmpq $3, %rcx37; AVX2-NEXT: movl $3, %eax38; AVX2-NEXT: cmovbq %rcx, %rax39; AVX2-NEXT: movl %r8d, -24(%rsp,%rax,4)40; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm041; AVX2-NEXT: retq42;43; AVX512F-LABEL: test_compress_v4i32:44; AVX512F: # %bb.0:45; AVX512F-NEXT: # kill: def $xmm2 killed $xmm2 def $zmm246; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm047; AVX512F-NEXT: vpslld $31, %xmm1, %xmm148; AVX512F-NEXT: vptestmd %zmm1, %zmm1, %k049; AVX512F-NEXT: kshiftlw $12, %k0, %k050; AVX512F-NEXT: kshiftrw $12, %k0, %k151; AVX512F-NEXT: vpcompressd %zmm0, %zmm2 {%k1}52; AVX512F-NEXT: vmovdqa %xmm2, %xmm053; AVX512F-NEXT: vzeroupper54; AVX512F-NEXT: retq55;56; AVX512VL-LABEL: test_compress_v4i32:57; AVX512VL: # %bb.0:58; AVX512VL-NEXT: vpslld $31, %xmm1, %xmm159; AVX512VL-NEXT: vptestmd %xmm1, %xmm1, %k160; AVX512VL-NEXT: vpcompressd %xmm0, %xmm2 {%k1}61; AVX512VL-NEXT: vmovdqa %xmm2, %xmm062; AVX512VL-NEXT: retq63 %out = call <4 x i32> @llvm.experimental.vector.compress(<4 x i32> %vec, <4 x i1> %mask, <4 x i32> %passthru)64 ret <4 x i32> %out65}66 67define <4 x float> @test_compress_v4f32(<4 x float> %vec, <4 x i1> %mask, <4 x float> %passthru) nounwind {68; AVX2-LABEL: test_compress_v4f32:69; AVX2: # %bb.0:70; AVX2-NEXT: vpslld $31, %xmm1, %xmm171; AVX2-NEXT: vpsrad $31, %xmm1, %xmm172; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)73; AVX2-NEXT: vpextrd $1, %xmm1, %edx74; AVX2-NEXT: vmovd %xmm1, %esi75; AVX2-NEXT: andl $1, %esi76; AVX2-NEXT: movl %esi, %edi77; AVX2-NEXT: subl %edx, %edi78; AVX2-NEXT: vpextrd $2, %xmm1, %ecx79; AVX2-NEXT: subl %ecx, %edi80; AVX2-NEXT: vpextrd $3, %xmm1, %eax81; AVX2-NEXT: subl %eax, %edi82; AVX2-NEXT: andl $3, %edi83; AVX2-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero84; AVX2-NEXT: vmovss %xmm0, -{{[0-9]+}}(%rsp)85; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rsi,4)86; AVX2-NEXT: andl $1, %edx87; AVX2-NEXT: addq %rsi, %rdx88; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rdx,4)89; AVX2-NEXT: andl $1, %ecx90; AVX2-NEXT: addq %rdx, %rcx91; AVX2-NEXT: andl $1, %eax92; AVX2-NEXT: addq %rcx, %rax93; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx94; AVX2-NEXT: andl $3, %ecx95; AVX2-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]96; AVX2-NEXT: vmovss %xmm0, -24(%rsp,%rcx,4)97; AVX2-NEXT: cmpq $3, %rax98; AVX2-NEXT: movl $3, %ecx99; AVX2-NEXT: cmovbq %rax, %rcx100; AVX2-NEXT: ja .LBB1_2101; AVX2-NEXT: # %bb.1:102; AVX2-NEXT: vmovaps %xmm1, %xmm0103; AVX2-NEXT: .LBB1_2:104; AVX2-NEXT: vmovss %xmm0, -24(%rsp,%rcx,4)105; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0106; AVX2-NEXT: retq107;108; AVX512F-LABEL: test_compress_v4f32:109; AVX512F: # %bb.0:110; AVX512F-NEXT: # kill: def $xmm2 killed $xmm2 def $zmm2111; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0112; AVX512F-NEXT: vpslld $31, %xmm1, %xmm1113; AVX512F-NEXT: vptestmd %zmm1, %zmm1, %k0114; AVX512F-NEXT: kshiftlw $12, %k0, %k0115; AVX512F-NEXT: kshiftrw $12, %k0, %k1116; AVX512F-NEXT: vcompressps %zmm0, %zmm2 {%k1}117; AVX512F-NEXT: vmovdqa %xmm2, %xmm0118; AVX512F-NEXT: vzeroupper119; AVX512F-NEXT: retq120;121; AVX512VL-LABEL: test_compress_v4f32:122; AVX512VL: # %bb.0:123; AVX512VL-NEXT: vpslld $31, %xmm1, %xmm1124; AVX512VL-NEXT: vptestmd %xmm1, %xmm1, %k1125; AVX512VL-NEXT: vcompressps %xmm0, %xmm2 {%k1}126; AVX512VL-NEXT: vmovdqa %xmm2, %xmm0127; AVX512VL-NEXT: retq128 %out = call <4 x float> @llvm.experimental.vector.compress(<4 x float> %vec, <4 x i1> %mask, <4 x float> %passthru)129 ret <4 x float> %out130}131 132define <2 x i64> @test_compress_v2i64(<2 x i64> %vec, <2 x i1> %mask, <2 x i64> %passthru) nounwind {133; AVX2-LABEL: test_compress_v2i64:134; AVX2: # %bb.0:135; AVX2-NEXT: vpsllq $63, %xmm1, %xmm1136; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3137; AVX2-NEXT: vpcmpgtq %xmm1, %xmm3, %xmm1138; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)139; AVX2-NEXT: vpextrq $1, %xmm1, %rax140; AVX2-NEXT: vmovq %xmm1, %rcx141; AVX2-NEXT: andl $1, %ecx142; AVX2-NEXT: movl %ecx, %edx143; AVX2-NEXT: subq %rax, %rcx144; AVX2-NEXT: movl %ecx, %eax145; AVX2-NEXT: andl $1, %eax146; AVX2-NEXT: vpextrq $1, %xmm0, %rsi147; AVX2-NEXT: cmpq $2, %rcx148; AVX2-NEXT: cmovbq -24(%rsp,%rax,8), %rsi149; AVX2-NEXT: vmovq %xmm0, -{{[0-9]+}}(%rsp)150; AVX2-NEXT: vpextrq $1, %xmm0, -24(%rsp,%rdx,8)151; AVX2-NEXT: cmpq $1, %rcx152; AVX2-NEXT: movl $1, %eax153; AVX2-NEXT: cmovbq %rcx, %rax154; AVX2-NEXT: movl %eax, %eax155; AVX2-NEXT: movq %rsi, -24(%rsp,%rax,8)156; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0157; AVX2-NEXT: retq158;159; AVX512F-LABEL: test_compress_v2i64:160; AVX512F: # %bb.0:161; AVX512F-NEXT: # kill: def $xmm2 killed $xmm2 def $zmm2162; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0163; AVX512F-NEXT: vpsllq $63, %xmm1, %xmm1164; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k0165; AVX512F-NEXT: kshiftlw $14, %k0, %k0166; AVX512F-NEXT: kshiftrw $14, %k0, %k1167; AVX512F-NEXT: vpcompressq %zmm0, %zmm2 {%k1}168; AVX512F-NEXT: vmovdqa %xmm2, %xmm0169; AVX512F-NEXT: vzeroupper170; AVX512F-NEXT: retq171;172; AVX512VL-LABEL: test_compress_v2i64:173; AVX512VL: # %bb.0:174; AVX512VL-NEXT: vpsllq $63, %xmm1, %xmm1175; AVX512VL-NEXT: vptestmq %xmm1, %xmm1, %k1176; AVX512VL-NEXT: vpcompressq %xmm0, %xmm2 {%k1}177; AVX512VL-NEXT: vmovdqa %xmm2, %xmm0178; AVX512VL-NEXT: retq179 %out = call <2 x i64> @llvm.experimental.vector.compress(<2 x i64> %vec, <2 x i1> %mask, <2 x i64> %passthru)180 ret <2 x i64> %out181}182 183define <2 x double> @test_compress_v2f64(<2 x double> %vec, <2 x i1> %mask, <2 x double> %passthru) nounwind {184; AVX2-LABEL: test_compress_v2f64:185; AVX2: # %bb.0:186; AVX2-NEXT: vpsllq $63, %xmm1, %xmm1187; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3188; AVX2-NEXT: vpcmpgtq %xmm1, %xmm3, %xmm1189; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)190; AVX2-NEXT: vpextrq $1, %xmm1, %rcx191; AVX2-NEXT: vmovq %xmm1, %rax192; AVX2-NEXT: andl $1, %eax193; AVX2-NEXT: movl %eax, %edx194; AVX2-NEXT: subq %rcx, %rax195; AVX2-NEXT: movl %eax, %ecx196; AVX2-NEXT: andl $1, %ecx197; AVX2-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero198; AVX2-NEXT: vmovlpd %xmm0, -{{[0-9]+}}(%rsp)199; AVX2-NEXT: vmovhpd %xmm0, -24(%rsp,%rdx,8)200; AVX2-NEXT: cmpq $2, %rax201; AVX2-NEXT: jb .LBB3_2202; AVX2-NEXT: # %bb.1:203; AVX2-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]204; AVX2-NEXT: .LBB3_2:205; AVX2-NEXT: cmpq $1, %rax206; AVX2-NEXT: movl $1, %ecx207; AVX2-NEXT: cmovbq %rax, %rcx208; AVX2-NEXT: movl %ecx, %eax209; AVX2-NEXT: vmovsd %xmm1, -24(%rsp,%rax,8)210; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0211; AVX2-NEXT: retq212;213; AVX512F-LABEL: test_compress_v2f64:214; AVX512F: # %bb.0:215; AVX512F-NEXT: # kill: def $xmm2 killed $xmm2 def $zmm2216; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0217; AVX512F-NEXT: vpsllq $63, %xmm1, %xmm1218; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k0219; AVX512F-NEXT: kshiftlw $14, %k0, %k0220; AVX512F-NEXT: kshiftrw $14, %k0, %k1221; AVX512F-NEXT: vcompresspd %zmm0, %zmm2 {%k1}222; AVX512F-NEXT: vmovdqa %xmm2, %xmm0223; AVX512F-NEXT: vzeroupper224; AVX512F-NEXT: retq225;226; AVX512VL-LABEL: test_compress_v2f64:227; AVX512VL: # %bb.0:228; AVX512VL-NEXT: vpsllq $63, %xmm1, %xmm1229; AVX512VL-NEXT: vptestmq %xmm1, %xmm1, %k1230; AVX512VL-NEXT: vcompresspd %xmm0, %xmm2 {%k1}231; AVX512VL-NEXT: vmovdqa %xmm2, %xmm0232; AVX512VL-NEXT: retq233 %out = call <2 x double> @llvm.experimental.vector.compress(<2 x double> %vec, <2 x i1> %mask, <2 x double> %passthru)234 ret <2 x double> %out235}236 237define <8 x i32> @test_compress_v8i32(<8 x i32> %vec, <8 x i1> %mask, <8 x i32> %passthru) nounwind {238; AVX2-LABEL: test_compress_v8i32:239; AVX2: # %bb.0:240; AVX2-NEXT: pushq %rbp241; AVX2-NEXT: movq %rsp, %rbp242; AVX2-NEXT: pushq %rbx243; AVX2-NEXT: andq $-32, %rsp244; AVX2-NEXT: subq $64, %rsp245; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero246; AVX2-NEXT: vpslld $31, %ymm1, %ymm1247; AVX2-NEXT: vpsrad $31, %ymm1, %ymm3248; AVX2-NEXT: vmovaps %ymm2, (%rsp)249; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm1250; AVX2-NEXT: vpackssdw %xmm1, %xmm3, %xmm2251; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero252; AVX2-NEXT: vpslld $31, %ymm2, %ymm2253; AVX2-NEXT: vpsrld $31, %ymm2, %ymm2254; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm4255; AVX2-NEXT: vpaddd %xmm4, %xmm2, %xmm2256; AVX2-NEXT: vpextrd $1, %xmm2, %eax257; AVX2-NEXT: vmovd %xmm2, %ecx258; AVX2-NEXT: addl %eax, %ecx259; AVX2-NEXT: vpextrd $2, %xmm2, %edx260; AVX2-NEXT: vpextrd $3, %xmm2, %eax261; AVX2-NEXT: addl %edx, %eax262; AVX2-NEXT: addl %ecx, %eax263; AVX2-NEXT: andl $7, %eax264; AVX2-NEXT: vpextrd $1, %xmm3, %ecx265; AVX2-NEXT: andl $1, %ecx266; AVX2-NEXT: vmovd %xmm3, %edx267; AVX2-NEXT: andl $1, %edx268; AVX2-NEXT: addq %rdx, %rcx269; AVX2-NEXT: vpextrd $2, %xmm3, %esi270; AVX2-NEXT: andl $1, %esi271; AVX2-NEXT: addq %rcx, %rsi272; AVX2-NEXT: vpextrd $3, %xmm3, %edi273; AVX2-NEXT: andl $1, %edi274; AVX2-NEXT: addq %rsi, %rdi275; AVX2-NEXT: vmovd %xmm1, %r8d276; AVX2-NEXT: andl $1, %r8d277; AVX2-NEXT: addq %rdi, %r8278; AVX2-NEXT: vpextrd $1, %xmm1, %r9d279; AVX2-NEXT: andl $1, %r9d280; AVX2-NEXT: addq %r8, %r9281; AVX2-NEXT: vpextrd $2, %xmm1, %r10d282; AVX2-NEXT: andl $1, %r10d283; AVX2-NEXT: addq %r9, %r10284; AVX2-NEXT: vpextrd $3, %xmm1, %r11d285; AVX2-NEXT: andl $1, %r11d286; AVX2-NEXT: addq %r10, %r11287; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm1288; AVX2-NEXT: vextractps $3, %xmm1, %ebx289; AVX2-NEXT: cmpq $8, %r11290; AVX2-NEXT: cmovbl (%rsp,%rax,4), %ebx291; AVX2-NEXT: vmovss %xmm0, (%rsp)292; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rdx,4)293; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)294; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rsi,4)295; AVX2-NEXT: andl $7, %edi296; AVX2-NEXT: vmovss %xmm1, (%rsp,%rdi,4)297; AVX2-NEXT: andl $7, %r8d298; AVX2-NEXT: vextractps $1, %xmm1, (%rsp,%r8,4)299; AVX2-NEXT: andl $7, %r9d300; AVX2-NEXT: vextractps $2, %xmm1, (%rsp,%r9,4)301; AVX2-NEXT: andl $7, %r10d302; AVX2-NEXT: vextractps $3, %xmm1, (%rsp,%r10,4)303; AVX2-NEXT: cmpq $7, %r11304; AVX2-NEXT: movl $7, %eax305; AVX2-NEXT: cmovbq %r11, %rax306; AVX2-NEXT: movl %eax, %eax307; AVX2-NEXT: movl %ebx, (%rsp,%rax,4)308; AVX2-NEXT: vmovaps (%rsp), %ymm0309; AVX2-NEXT: leaq -8(%rbp), %rsp310; AVX2-NEXT: popq %rbx311; AVX2-NEXT: popq %rbp312; AVX2-NEXT: retq313;314; AVX512F-LABEL: test_compress_v8i32:315; AVX512F: # %bb.0:316; AVX512F-NEXT: # kill: def $ymm2 killed $ymm2 def $zmm2317; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0318; AVX512F-NEXT: vpmovsxwq %xmm1, %zmm1319; AVX512F-NEXT: vpsllq $63, %zmm1, %zmm1320; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1321; AVX512F-NEXT: vpcompressd %zmm0, %zmm2 {%k1}322; AVX512F-NEXT: vmovdqa %ymm2, %ymm0323; AVX512F-NEXT: retq324;325; AVX512VL-LABEL: test_compress_v8i32:326; AVX512VL: # %bb.0:327; AVX512VL-NEXT: vpsllw $15, %xmm1, %xmm1328; AVX512VL-NEXT: vpmovw2m %xmm1, %k1329; AVX512VL-NEXT: vpcompressd %ymm0, %ymm2 {%k1}330; AVX512VL-NEXT: vmovdqa %ymm2, %ymm0331; AVX512VL-NEXT: retq332 %out = call <8 x i32> @llvm.experimental.vector.compress(<8 x i32> %vec, <8 x i1> %mask, <8 x i32> %passthru)333 ret <8 x i32> %out334}335 336define <8 x float> @test_compress_v8f32(<8 x float> %vec, <8 x i1> %mask, <8 x float> %passthru) nounwind {337; AVX2-LABEL: test_compress_v8f32:338; AVX2: # %bb.0:339; AVX2-NEXT: pushq %rbp340; AVX2-NEXT: movq %rsp, %rbp341; AVX2-NEXT: andq $-32, %rsp342; AVX2-NEXT: subq $64, %rsp343; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero344; AVX2-NEXT: vpslld $31, %ymm1, %ymm1345; AVX2-NEXT: vpsrad $31, %ymm1, %ymm3346; AVX2-NEXT: vmovaps %ymm2, (%rsp)347; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm1348; AVX2-NEXT: vpackssdw %xmm1, %xmm3, %xmm2349; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero350; AVX2-NEXT: vpslld $31, %ymm2, %ymm2351; AVX2-NEXT: vpsrld $31, %ymm2, %ymm2352; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm4353; AVX2-NEXT: vpaddd %xmm4, %xmm2, %xmm2354; AVX2-NEXT: vpextrd $1, %xmm2, %eax355; AVX2-NEXT: vmovd %xmm2, %ecx356; AVX2-NEXT: addl %eax, %ecx357; AVX2-NEXT: vpextrd $2, %xmm2, %eax358; AVX2-NEXT: vpextrd $3, %xmm2, %edx359; AVX2-NEXT: addl %eax, %edx360; AVX2-NEXT: addl %ecx, %edx361; AVX2-NEXT: andl $7, %edx362; AVX2-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero363; AVX2-NEXT: vmovss %xmm0, (%rsp)364; AVX2-NEXT: vmovd %xmm3, %eax365; AVX2-NEXT: andl $1, %eax366; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rax,4)367; AVX2-NEXT: vpextrd $1, %xmm3, %ecx368; AVX2-NEXT: andl $1, %ecx369; AVX2-NEXT: addq %rax, %rcx370; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)371; AVX2-NEXT: vpextrd $2, %xmm3, %eax372; AVX2-NEXT: andl $1, %eax373; AVX2-NEXT: addq %rcx, %rax374; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rax,4)375; AVX2-NEXT: vpextrd $3, %xmm3, %ecx376; AVX2-NEXT: andl $1, %ecx377; AVX2-NEXT: addq %rax, %rcx378; AVX2-NEXT: vmovd %xmm1, %eax379; AVX2-NEXT: andl $1, %eax380; AVX2-NEXT: addq %rcx, %rax381; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx382; AVX2-NEXT: andl $7, %ecx383; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0384; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)385; AVX2-NEXT: vpextrd $1, %xmm1, %ecx386; AVX2-NEXT: andl $1, %ecx387; AVX2-NEXT: addq %rax, %rcx388; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax389; AVX2-NEXT: andl $7, %eax390; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rax,4)391; AVX2-NEXT: vpextrd $2, %xmm1, %edx392; AVX2-NEXT: andl $1, %edx393; AVX2-NEXT: addq %rcx, %rdx394; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx395; AVX2-NEXT: andl $7, %ecx396; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)397; AVX2-NEXT: vpextrd $3, %xmm1, %eax398; AVX2-NEXT: andl $1, %eax399; AVX2-NEXT: addq %rdx, %rax400; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx401; AVX2-NEXT: andl $7, %edx402; AVX2-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]403; AVX2-NEXT: vmovss %xmm0, (%rsp,%rdx,4)404; AVX2-NEXT: cmpq $8, %rax405; AVX2-NEXT: jae .LBB5_2406; AVX2-NEXT: # %bb.1:407; AVX2-NEXT: vmovaps %xmm2, %xmm0408; AVX2-NEXT: .LBB5_2:409; AVX2-NEXT: cmpq $7, %rax410; AVX2-NEXT: movl $7, %ecx411; AVX2-NEXT: cmovbq %rax, %rcx412; AVX2-NEXT: movl %ecx, %eax413; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax,4)414; AVX2-NEXT: vmovaps (%rsp), %ymm0415; AVX2-NEXT: movq %rbp, %rsp416; AVX2-NEXT: popq %rbp417; AVX2-NEXT: retq418;419; AVX512F-LABEL: test_compress_v8f32:420; AVX512F: # %bb.0:421; AVX512F-NEXT: # kill: def $ymm2 killed $ymm2 def $zmm2422; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0423; AVX512F-NEXT: vpmovsxwq %xmm1, %zmm1424; AVX512F-NEXT: vpsllq $63, %zmm1, %zmm1425; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1426; AVX512F-NEXT: vcompressps %zmm0, %zmm2 {%k1}427; AVX512F-NEXT: vmovdqa %ymm2, %ymm0428; AVX512F-NEXT: retq429;430; AVX512VL-LABEL: test_compress_v8f32:431; AVX512VL: # %bb.0:432; AVX512VL-NEXT: vpsllw $15, %xmm1, %xmm1433; AVX512VL-NEXT: vpmovw2m %xmm1, %k1434; AVX512VL-NEXT: vcompressps %ymm0, %ymm2 {%k1}435; AVX512VL-NEXT: vmovdqa %ymm2, %ymm0436; AVX512VL-NEXT: retq437 %out = call <8 x float> @llvm.experimental.vector.compress(<8 x float> %vec, <8 x i1> %mask, <8 x float> %passthru)438 ret <8 x float> %out439}440 441define <4 x i64> @test_compress_v4i64(<4 x i64> %vec, <4 x i1> %mask, <4 x i64> %passthru) nounwind {442; AVX2-LABEL: test_compress_v4i64:443; AVX2: # %bb.0:444; AVX2-NEXT: pushq %rbp445; AVX2-NEXT: movq %rsp, %rbp446; AVX2-NEXT: andq $-32, %rsp447; AVX2-NEXT: subq $64, %rsp448; AVX2-NEXT: vpslld $31, %xmm1, %xmm1449; AVX2-NEXT: vpsrad $31, %xmm1, %xmm1450; AVX2-NEXT: vpmovsxdq %xmm1, %ymm1451; AVX2-NEXT: vmovaps %ymm2, (%rsp)452; AVX2-NEXT: vpsrlq $63, %ymm1, %ymm2453; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3454; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2455; AVX2-NEXT: vpextrq $1, %xmm2, %rcx456; AVX2-NEXT: vmovq %xmm2, %rax457; AVX2-NEXT: addl %ecx, %eax458; AVX2-NEXT: andl $3, %eax459; AVX2-NEXT: vpextrq $1, %xmm1, %rcx460; AVX2-NEXT: vmovq %xmm1, %rdx461; AVX2-NEXT: andl $1, %edx462; AVX2-NEXT: movl %edx, %esi463; AVX2-NEXT: subq %rcx, %rdx464; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1465; AVX2-NEXT: vmovq %xmm1, %rcx466; AVX2-NEXT: movl %edx, %edi467; AVX2-NEXT: subq %rcx, %rdx468; AVX2-NEXT: vpextrq $1, %xmm1, %rcx469; AVX2-NEXT: movq %rdx, %r8470; AVX2-NEXT: subq %rcx, %r8471; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1472; AVX2-NEXT: vpextrq $1, %xmm1, %rcx473; AVX2-NEXT: cmpq $4, %r8474; AVX2-NEXT: cmovbq (%rsp,%rax,8), %rcx475; AVX2-NEXT: vmovq %xmm0, (%rsp)476; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rsi,8)477; AVX2-NEXT: vmovq %xmm1, (%rsp,%rdi,8)478; AVX2-NEXT: andl $3, %edx479; AVX2-NEXT: vpextrq $1, %xmm1, (%rsp,%rdx,8)480; AVX2-NEXT: cmpq $3, %r8481; AVX2-NEXT: movl $3, %eax482; AVX2-NEXT: cmovbq %r8, %rax483; AVX2-NEXT: movl %eax, %eax484; AVX2-NEXT: movq %rcx, (%rsp,%rax,8)485; AVX2-NEXT: vmovaps (%rsp), %ymm0486; AVX2-NEXT: movq %rbp, %rsp487; AVX2-NEXT: popq %rbp488; AVX2-NEXT: retq489;490; AVX512F-LABEL: test_compress_v4i64:491; AVX512F: # %bb.0:492; AVX512F-NEXT: # kill: def $ymm2 killed $ymm2 def $zmm2493; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0494; AVX512F-NEXT: vpslld $31, %xmm1, %xmm1495; AVX512F-NEXT: vptestmd %zmm1, %zmm1, %k0496; AVX512F-NEXT: kshiftlw $12, %k0, %k0497; AVX512F-NEXT: kshiftrw $12, %k0, %k1498; AVX512F-NEXT: vpcompressq %zmm0, %zmm2 {%k1}499; AVX512F-NEXT: vmovdqa %ymm2, %ymm0500; AVX512F-NEXT: retq501;502; AVX512VL-LABEL: test_compress_v4i64:503; AVX512VL: # %bb.0:504; AVX512VL-NEXT: vpslld $31, %xmm1, %xmm1505; AVX512VL-NEXT: vptestmd %xmm1, %xmm1, %k1506; AVX512VL-NEXT: vpcompressq %ymm0, %ymm2 {%k1}507; AVX512VL-NEXT: vmovdqa %ymm2, %ymm0508; AVX512VL-NEXT: retq509 %out = call <4 x i64> @llvm.experimental.vector.compress(<4 x i64> %vec, <4 x i1> %mask, <4 x i64> %passthru)510 ret <4 x i64> %out511}512 513define <4 x double> @test_compress_v4f64(<4 x double> %vec, <4 x i1> %mask, <4 x double> %passthru) nounwind {514; AVX2-LABEL: test_compress_v4f64:515; AVX2: # %bb.0:516; AVX2-NEXT: pushq %rbp517; AVX2-NEXT: movq %rsp, %rbp518; AVX2-NEXT: andq $-32, %rsp519; AVX2-NEXT: subq $64, %rsp520; AVX2-NEXT: vpslld $31, %xmm1, %xmm1521; AVX2-NEXT: vpsrad $31, %xmm1, %xmm1522; AVX2-NEXT: vpmovsxdq %xmm1, %ymm3523; AVX2-NEXT: vmovaps %ymm2, (%rsp)524; AVX2-NEXT: vpsrlq $63, %ymm3, %ymm1525; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2526; AVX2-NEXT: vpaddq %xmm2, %xmm1, %xmm1527; AVX2-NEXT: vpextrq $1, %xmm1, %rax528; AVX2-NEXT: vmovq %xmm1, %rcx529; AVX2-NEXT: addl %eax, %ecx530; AVX2-NEXT: andl $3, %ecx531; AVX2-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero532; AVX2-NEXT: vmovlpd %xmm0, (%rsp)533; AVX2-NEXT: vmovq %xmm3, %rax534; AVX2-NEXT: andl $1, %eax535; AVX2-NEXT: movl %eax, %ecx536; AVX2-NEXT: vmovhpd %xmm0, (%rsp,%rcx,8)537; AVX2-NEXT: vpextrq $1, %xmm3, %rcx538; AVX2-NEXT: subq %rcx, %rax539; AVX2-NEXT: movl %eax, %ecx540; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0541; AVX2-NEXT: vmovlpd %xmm0, (%rsp,%rcx,8)542; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm2543; AVX2-NEXT: vmovq %xmm2, %rcx544; AVX2-NEXT: subq %rcx, %rax545; AVX2-NEXT: movl %eax, %ecx546; AVX2-NEXT: andl $3, %ecx547; AVX2-NEXT: vmovhpd %xmm0, (%rsp,%rcx,8)548; AVX2-NEXT: vpextrq $1, %xmm2, %rcx549; AVX2-NEXT: subq %rcx, %rax550; AVX2-NEXT: cmpq $4, %rax551; AVX2-NEXT: jb .LBB7_2552; AVX2-NEXT: # %bb.1:553; AVX2-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]554; AVX2-NEXT: .LBB7_2:555; AVX2-NEXT: cmpq $3, %rax556; AVX2-NEXT: movl $3, %ecx557; AVX2-NEXT: cmovbq %rax, %rcx558; AVX2-NEXT: movl %ecx, %eax559; AVX2-NEXT: vmovsd %xmm1, (%rsp,%rax,8)560; AVX2-NEXT: vmovaps (%rsp), %ymm0561; AVX2-NEXT: movq %rbp, %rsp562; AVX2-NEXT: popq %rbp563; AVX2-NEXT: retq564;565; AVX512F-LABEL: test_compress_v4f64:566; AVX512F: # %bb.0:567; AVX512F-NEXT: # kill: def $ymm2 killed $ymm2 def $zmm2568; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0569; AVX512F-NEXT: vpslld $31, %xmm1, %xmm1570; AVX512F-NEXT: vptestmd %zmm1, %zmm1, %k0571; AVX512F-NEXT: kshiftlw $12, %k0, %k0572; AVX512F-NEXT: kshiftrw $12, %k0, %k1573; AVX512F-NEXT: vcompresspd %zmm0, %zmm2 {%k1}574; AVX512F-NEXT: vmovdqa %ymm2, %ymm0575; AVX512F-NEXT: retq576;577; AVX512VL-LABEL: test_compress_v4f64:578; AVX512VL: # %bb.0:579; AVX512VL-NEXT: vpslld $31, %xmm1, %xmm1580; AVX512VL-NEXT: vptestmd %xmm1, %xmm1, %k1581; AVX512VL-NEXT: vcompresspd %ymm0, %ymm2 {%k1}582; AVX512VL-NEXT: vmovdqa %ymm2, %ymm0583; AVX512VL-NEXT: retq584 %out = call <4 x double> @llvm.experimental.vector.compress(<4 x double> %vec, <4 x i1> %mask, <4 x double> %passthru)585 ret <4 x double> %out586}587 588define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x i32> %passthru) nounwind {589; AVX2-LABEL: test_compress_v16i32:590; AVX2: # %bb.0:591; AVX2-NEXT: pushq %rbp592; AVX2-NEXT: movq %rsp, %rbp593; AVX2-NEXT: pushq %r15594; AVX2-NEXT: pushq %r14595; AVX2-NEXT: pushq %r13596; AVX2-NEXT: pushq %r12597; AVX2-NEXT: pushq %rbx598; AVX2-NEXT: andq $-32, %rsp599; AVX2-NEXT: subq $128, %rsp600; AVX2-NEXT: vmovaps %ymm4, {{[0-9]+}}(%rsp)601; AVX2-NEXT: vmovaps %ymm3, (%rsp)602; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm3603; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero604; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]605; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero606; AVX2-NEXT: vpaddd %ymm3, %ymm4, %ymm3607; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4608; AVX2-NEXT: vpaddd %xmm4, %xmm3, %xmm3609; AVX2-NEXT: vpextrd $1, %xmm3, %eax610; AVX2-NEXT: vmovd %xmm3, %ecx611; AVX2-NEXT: addl %eax, %ecx612; AVX2-NEXT: vpextrd $2, %xmm3, %eax613; AVX2-NEXT: vpextrd $3, %xmm3, %edx614; AVX2-NEXT: addl %eax, %edx615; AVX2-NEXT: addl %ecx, %edx616; AVX2-NEXT: andl $15, %edx617; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill618; AVX2-NEXT: vpextrb $1, %xmm2, %eax619; AVX2-NEXT: andl $1, %eax620; AVX2-NEXT: vmovd %xmm2, %ecx621; AVX2-NEXT: andl $1, %ecx622; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill623; AVX2-NEXT: addq %rcx, %rax624; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill625; AVX2-NEXT: vpextrb $2, %xmm2, %ecx626; AVX2-NEXT: andl $1, %ecx627; AVX2-NEXT: addq %rax, %rcx628; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill629; AVX2-NEXT: vpextrb $3, %xmm2, %eax630; AVX2-NEXT: andl $1, %eax631; AVX2-NEXT: addq %rcx, %rax632; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill633; AVX2-NEXT: vpextrb $4, %xmm2, %r8d634; AVX2-NEXT: andl $1, %r8d635; AVX2-NEXT: addq %rax, %r8636; AVX2-NEXT: vpextrb $5, %xmm2, %r9d637; AVX2-NEXT: andl $1, %r9d638; AVX2-NEXT: addq %r8, %r9639; AVX2-NEXT: vpextrb $6, %xmm2, %r10d640; AVX2-NEXT: andl $1, %r10d641; AVX2-NEXT: addq %r9, %r10642; AVX2-NEXT: vpextrb $7, %xmm2, %r11d643; AVX2-NEXT: andl $1, %r11d644; AVX2-NEXT: addq %r10, %r11645; AVX2-NEXT: vpextrb $8, %xmm2, %ebx646; AVX2-NEXT: andl $1, %ebx647; AVX2-NEXT: addq %r11, %rbx648; AVX2-NEXT: vpextrb $9, %xmm2, %r14d649; AVX2-NEXT: andl $1, %r14d650; AVX2-NEXT: addq %rbx, %r14651; AVX2-NEXT: vpextrb $10, %xmm2, %r15d652; AVX2-NEXT: andl $1, %r15d653; AVX2-NEXT: addq %r14, %r15654; AVX2-NEXT: vpextrb $11, %xmm2, %r12d655; AVX2-NEXT: andl $1, %r12d656; AVX2-NEXT: addq %r15, %r12657; AVX2-NEXT: vpextrb $12, %xmm2, %r13d658; AVX2-NEXT: andl $1, %r13d659; AVX2-NEXT: addq %r12, %r13660; AVX2-NEXT: vpextrb $13, %xmm2, %ecx661; AVX2-NEXT: andl $1, %ecx662; AVX2-NEXT: addq %r13, %rcx663; AVX2-NEXT: vpextrb $14, %xmm2, %eax664; AVX2-NEXT: andl $1, %eax665; AVX2-NEXT: addq %rcx, %rax666; AVX2-NEXT: vpextrb $15, %xmm2, %edx667; AVX2-NEXT: andl $1, %edx668; AVX2-NEXT: addq %rax, %rdx669; AVX2-NEXT: vextractf128 $1, %ymm1, %xmm2670; AVX2-NEXT: cmpq $16, %rdx671; AVX2-NEXT: vextractps $3, %xmm2, %esi672; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload673; AVX2-NEXT: cmovbl (%rsp,%rdi,4), %esi674; AVX2-NEXT: movl %esi, %edi675; AVX2-NEXT: vmovss %xmm0, (%rsp)676; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload677; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rsi,4)678; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload679; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rsi,4)680; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload681; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rsi,4)682; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0683; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload684; AVX2-NEXT: vmovss %xmm0, (%rsp,%rsi,4)685; AVX2-NEXT: andl $15, %r8d686; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%r8,4)687; AVX2-NEXT: andl $15, %r9d688; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%r9,4)689; AVX2-NEXT: andl $15, %r10d690; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%r10,4)691; AVX2-NEXT: andl $15, %r11d692; AVX2-NEXT: vmovss %xmm1, (%rsp,%r11,4)693; AVX2-NEXT: andl $15, %ebx694; AVX2-NEXT: vextractps $1, %xmm1, (%rsp,%rbx,4)695; AVX2-NEXT: andl $15, %r14d696; AVX2-NEXT: vextractps $2, %xmm1, (%rsp,%r14,4)697; AVX2-NEXT: andl $15, %r15d698; AVX2-NEXT: vextractps $3, %xmm1, (%rsp,%r15,4)699; AVX2-NEXT: andl $15, %r12d700; AVX2-NEXT: vmovss %xmm2, (%rsp,%r12,4)701; AVX2-NEXT: andl $15, %r13d702; AVX2-NEXT: vextractps $1, %xmm2, (%rsp,%r13,4)703; AVX2-NEXT: andl $15, %ecx704; AVX2-NEXT: vextractps $2, %xmm2, (%rsp,%rcx,4)705; AVX2-NEXT: andl $15, %eax706; AVX2-NEXT: vextractps $3, %xmm2, (%rsp,%rax,4)707; AVX2-NEXT: cmpq $15, %rdx708; AVX2-NEXT: movl $15, %eax709; AVX2-NEXT: cmovbq %rdx, %rax710; AVX2-NEXT: movl %eax, %eax711; AVX2-NEXT: movl %edi, (%rsp,%rax,4)712; AVX2-NEXT: vmovaps (%rsp), %ymm0713; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm1714; AVX2-NEXT: leaq -40(%rbp), %rsp715; AVX2-NEXT: popq %rbx716; AVX2-NEXT: popq %r12717; AVX2-NEXT: popq %r13718; AVX2-NEXT: popq %r14719; AVX2-NEXT: popq %r15720; AVX2-NEXT: popq %rbp721; AVX2-NEXT: retq722;723; AVX512F-LABEL: test_compress_v16i32:724; AVX512F: # %bb.0:725; AVX512F-NEXT: vpmovsxbd %xmm1, %zmm1726; AVX512F-NEXT: vpslld $31, %zmm1, %zmm1727; AVX512F-NEXT: vptestmd %zmm1, %zmm1, %k1728; AVX512F-NEXT: vpcompressd %zmm0, %zmm2 {%k1}729; AVX512F-NEXT: vmovdqa64 %zmm2, %zmm0730; AVX512F-NEXT: retq731;732; AVX512VL-LABEL: test_compress_v16i32:733; AVX512VL: # %bb.0:734; AVX512VL-NEXT: vpsllw $7, %xmm1, %xmm1735; AVX512VL-NEXT: vpmovb2m %xmm1, %k1736; AVX512VL-NEXT: vpcompressd %zmm0, %zmm2 {%k1}737; AVX512VL-NEXT: vmovdqa64 %zmm2, %zmm0738; AVX512VL-NEXT: retq739 %out = call <16 x i32> @llvm.experimental.vector.compress(<16 x i32> %vec, <16 x i1> %mask, <16 x i32> %passthru)740 ret <16 x i32> %out741}742 743define <16 x float> @test_compress_v16f32(<16 x float> %vec, <16 x i1> %mask, <16 x float> %passthru) nounwind {744; AVX2-LABEL: test_compress_v16f32:745; AVX2: # %bb.0:746; AVX2-NEXT: pushq %rbp747; AVX2-NEXT: movq %rsp, %rbp748; AVX2-NEXT: andq $-32, %rsp749; AVX2-NEXT: subq $96, %rsp750; AVX2-NEXT: vmovaps %ymm4, {{[0-9]+}}(%rsp)751; AVX2-NEXT: vmovaps %ymm3, (%rsp)752; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm3753; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero754; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]755; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero756; AVX2-NEXT: vpaddd %ymm3, %ymm4, %ymm3757; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4758; AVX2-NEXT: vpaddd %xmm4, %xmm3, %xmm3759; AVX2-NEXT: vpextrd $1, %xmm3, %eax760; AVX2-NEXT: vmovd %xmm3, %ecx761; AVX2-NEXT: addl %eax, %ecx762; AVX2-NEXT: vpextrd $2, %xmm3, %eax763; AVX2-NEXT: vpextrd $3, %xmm3, %edx764; AVX2-NEXT: addl %eax, %edx765; AVX2-NEXT: addl %ecx, %edx766; AVX2-NEXT: andl $15, %edx767; AVX2-NEXT: vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero768; AVX2-NEXT: vmovss %xmm0, (%rsp)769; AVX2-NEXT: vmovd %xmm2, %eax770; AVX2-NEXT: andl $1, %eax771; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rax,4)772; AVX2-NEXT: vpextrb $1, %xmm2, %ecx773; AVX2-NEXT: andl $1, %ecx774; AVX2-NEXT: addq %rax, %rcx775; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)776; AVX2-NEXT: vpextrb $2, %xmm2, %eax777; AVX2-NEXT: andl $1, %eax778; AVX2-NEXT: addq %rcx, %rax779; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rax,4)780; AVX2-NEXT: vpextrb $3, %xmm2, %ecx781; AVX2-NEXT: andl $1, %ecx782; AVX2-NEXT: addq %rax, %rcx783; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0784; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)785; AVX2-NEXT: vpextrb $4, %xmm2, %eax786; AVX2-NEXT: andl $1, %eax787; AVX2-NEXT: addq %rcx, %rax788; AVX2-NEXT: vpextrb $5, %xmm2, %ecx789; AVX2-NEXT: andl $1, %ecx790; AVX2-NEXT: addq %rax, %rcx791; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax792; AVX2-NEXT: andl $15, %eax793; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rax,4)794; AVX2-NEXT: vpextrb $6, %xmm2, %eax795; AVX2-NEXT: andl $1, %eax796; AVX2-NEXT: addq %rcx, %rax797; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx798; AVX2-NEXT: andl $15, %ecx799; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)800; AVX2-NEXT: vpextrb $7, %xmm2, %ecx801; AVX2-NEXT: andl $1, %ecx802; AVX2-NEXT: addq %rax, %rcx803; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax804; AVX2-NEXT: andl $15, %eax805; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rax,4)806; AVX2-NEXT: vpextrb $8, %xmm2, %eax807; AVX2-NEXT: andl $1, %eax808; AVX2-NEXT: addq %rcx, %rax809; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx810; AVX2-NEXT: andl $15, %ecx811; AVX2-NEXT: vmovss %xmm1, (%rsp,%rcx,4)812; AVX2-NEXT: vpextrb $9, %xmm2, %ecx813; AVX2-NEXT: andl $1, %ecx814; AVX2-NEXT: addq %rax, %rcx815; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax816; AVX2-NEXT: andl $15, %eax817; AVX2-NEXT: vextractps $1, %xmm1, (%rsp,%rax,4)818; AVX2-NEXT: vpextrb $10, %xmm2, %eax819; AVX2-NEXT: andl $1, %eax820; AVX2-NEXT: addq %rcx, %rax821; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx822; AVX2-NEXT: andl $15, %ecx823; AVX2-NEXT: vextractps $2, %xmm1, (%rsp,%rcx,4)824; AVX2-NEXT: vpextrb $11, %xmm2, %ecx825; AVX2-NEXT: andl $1, %ecx826; AVX2-NEXT: addq %rax, %rcx827; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax828; AVX2-NEXT: andl $15, %eax829; AVX2-NEXT: vextractps $3, %xmm1, (%rsp,%rax,4)830; AVX2-NEXT: vpextrb $12, %xmm2, %eax831; AVX2-NEXT: andl $1, %eax832; AVX2-NEXT: addq %rcx, %rax833; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx834; AVX2-NEXT: andl $15, %ecx835; AVX2-NEXT: vextractf128 $1, %ymm1, %xmm0836; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)837; AVX2-NEXT: vpextrb $13, %xmm2, %ecx838; AVX2-NEXT: andl $1, %ecx839; AVX2-NEXT: addq %rax, %rcx840; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax841; AVX2-NEXT: andl $15, %eax842; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rax,4)843; AVX2-NEXT: vpextrb $14, %xmm2, %edx844; AVX2-NEXT: andl $1, %edx845; AVX2-NEXT: addq %rcx, %rdx846; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx847; AVX2-NEXT: andl $15, %ecx848; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)849; AVX2-NEXT: vpextrb $15, %xmm2, %eax850; AVX2-NEXT: andl $1, %eax851; AVX2-NEXT: addq %rdx, %rax852; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx853; AVX2-NEXT: andl $15, %edx854; AVX2-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]855; AVX2-NEXT: vmovss %xmm0, (%rsp,%rdx,4)856; AVX2-NEXT: cmpq $16, %rax857; AVX2-NEXT: jae .LBB9_2858; AVX2-NEXT: # %bb.1:859; AVX2-NEXT: vmovaps %xmm3, %xmm0860; AVX2-NEXT: .LBB9_2:861; AVX2-NEXT: cmpq $15, %rax862; AVX2-NEXT: movl $15, %ecx863; AVX2-NEXT: cmovbq %rax, %rcx864; AVX2-NEXT: movl %ecx, %eax865; AVX2-NEXT: vmovss %xmm0, (%rsp,%rax,4)866; AVX2-NEXT: vmovaps (%rsp), %ymm0867; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm1868; AVX2-NEXT: movq %rbp, %rsp869; AVX2-NEXT: popq %rbp870; AVX2-NEXT: retq871;872; AVX512F-LABEL: test_compress_v16f32:873; AVX512F: # %bb.0:874; AVX512F-NEXT: vpmovsxbd %xmm1, %zmm1875; AVX512F-NEXT: vpslld $31, %zmm1, %zmm1876; AVX512F-NEXT: vptestmd %zmm1, %zmm1, %k1877; AVX512F-NEXT: vcompressps %zmm0, %zmm2 {%k1}878; AVX512F-NEXT: vmovdqa64 %zmm2, %zmm0879; AVX512F-NEXT: retq880;881; AVX512VL-LABEL: test_compress_v16f32:882; AVX512VL: # %bb.0:883; AVX512VL-NEXT: vpsllw $7, %xmm1, %xmm1884; AVX512VL-NEXT: vpmovb2m %xmm1, %k1885; AVX512VL-NEXT: vcompressps %zmm0, %zmm2 {%k1}886; AVX512VL-NEXT: vmovdqa64 %zmm2, %zmm0887; AVX512VL-NEXT: retq888 %out = call <16 x float> @llvm.experimental.vector.compress(<16 x float> %vec, <16 x i1> %mask, <16 x float> %passthru)889 ret <16 x float> %out890}891 892define <8 x i64> @test_compress_v8i64(<8 x i64> %vec, <8 x i1> %mask, <8 x i64> %passthru) nounwind {893; AVX2-LABEL: test_compress_v8i64:894; AVX2: # %bb.0:895; AVX2-NEXT: pushq %rbp896; AVX2-NEXT: movq %rsp, %rbp897; AVX2-NEXT: pushq %rbx898; AVX2-NEXT: andq $-32, %rsp899; AVX2-NEXT: subq $96, %rsp900; AVX2-NEXT: vmovaps %ymm4, {{[0-9]+}}(%rsp)901; AVX2-NEXT: vmovaps %ymm3, (%rsp)902; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm3903; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm4 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero904; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]905; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero906; AVX2-NEXT: vpaddq %ymm3, %ymm4, %ymm3907; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4908; AVX2-NEXT: vpaddq %xmm4, %xmm3, %xmm3909; AVX2-NEXT: vpextrq $1, %xmm3, %rcx910; AVX2-NEXT: vmovq %xmm3, %rax911; AVX2-NEXT: addl %ecx, %eax912; AVX2-NEXT: andl $7, %eax913; AVX2-NEXT: vpextrw $1, %xmm2, %ecx914; AVX2-NEXT: andl $1, %ecx915; AVX2-NEXT: vmovd %xmm2, %edx916; AVX2-NEXT: andl $1, %edx917; AVX2-NEXT: addq %rdx, %rcx918; AVX2-NEXT: vpextrw $2, %xmm2, %esi919; AVX2-NEXT: andl $1, %esi920; AVX2-NEXT: addq %rcx, %rsi921; AVX2-NEXT: vpextrw $3, %xmm2, %edi922; AVX2-NEXT: andl $1, %edi923; AVX2-NEXT: addq %rsi, %rdi924; AVX2-NEXT: vpextrw $4, %xmm2, %r8d925; AVX2-NEXT: andl $1, %r8d926; AVX2-NEXT: addq %rdi, %r8927; AVX2-NEXT: vpextrw $5, %xmm2, %r9d928; AVX2-NEXT: andl $1, %r9d929; AVX2-NEXT: addq %r8, %r9930; AVX2-NEXT: vpextrw $6, %xmm2, %r10d931; AVX2-NEXT: andl $1, %r10d932; AVX2-NEXT: addq %r9, %r10933; AVX2-NEXT: vpextrw $7, %xmm2, %r11d934; AVX2-NEXT: andl $1, %r11d935; AVX2-NEXT: addq %r10, %r11936; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2937; AVX2-NEXT: vpextrq $1, %xmm2, %rbx938; AVX2-NEXT: cmpq $8, %r11939; AVX2-NEXT: cmovbq (%rsp,%rax,8), %rbx940; AVX2-NEXT: vmovq %xmm0, (%rsp)941; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rdx,8)942; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0943; AVX2-NEXT: vmovq %xmm0, (%rsp,%rcx,8)944; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rsi,8)945; AVX2-NEXT: andl $7, %edi946; AVX2-NEXT: vmovq %xmm1, (%rsp,%rdi,8)947; AVX2-NEXT: andl $7, %r8d948; AVX2-NEXT: vpextrq $1, %xmm1, (%rsp,%r8,8)949; AVX2-NEXT: andl $7, %r9d950; AVX2-NEXT: vmovq %xmm2, (%rsp,%r9,8)951; AVX2-NEXT: andl $7, %r10d952; AVX2-NEXT: vpextrq $1, %xmm2, (%rsp,%r10,8)953; AVX2-NEXT: cmpq $7, %r11954; AVX2-NEXT: movl $7, %eax955; AVX2-NEXT: cmovbq %r11, %rax956; AVX2-NEXT: movl %eax, %eax957; AVX2-NEXT: movq %rbx, (%rsp,%rax,8)958; AVX2-NEXT: vmovaps (%rsp), %ymm0959; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm1960; AVX2-NEXT: leaq -8(%rbp), %rsp961; AVX2-NEXT: popq %rbx962; AVX2-NEXT: popq %rbp963; AVX2-NEXT: retq964;965; AVX512F-LABEL: test_compress_v8i64:966; AVX512F: # %bb.0:967; AVX512F-NEXT: vpmovsxwq %xmm1, %zmm1968; AVX512F-NEXT: vpsllq $63, %zmm1, %zmm1969; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1970; AVX512F-NEXT: vpcompressq %zmm0, %zmm2 {%k1}971; AVX512F-NEXT: vmovdqa64 %zmm2, %zmm0972; AVX512F-NEXT: retq973;974; AVX512VL-LABEL: test_compress_v8i64:975; AVX512VL: # %bb.0:976; AVX512VL-NEXT: vpsllw $15, %xmm1, %xmm1977; AVX512VL-NEXT: vpmovw2m %xmm1, %k1978; AVX512VL-NEXT: vpcompressq %zmm0, %zmm2 {%k1}979; AVX512VL-NEXT: vmovdqa64 %zmm2, %zmm0980; AVX512VL-NEXT: retq981 %out = call <8 x i64> @llvm.experimental.vector.compress(<8 x i64> %vec, <8 x i1> %mask, <8 x i64> %passthru)982 ret <8 x i64> %out983}984 985define <8 x double> @test_compress_v8f64(<8 x double> %vec, <8 x i1> %mask, <8 x double> %passthru) nounwind {986; AVX2-LABEL: test_compress_v8f64:987; AVX2: # %bb.0:988; AVX2-NEXT: pushq %rbp989; AVX2-NEXT: movq %rsp, %rbp990; AVX2-NEXT: andq $-32, %rsp991; AVX2-NEXT: subq $96, %rsp992; AVX2-NEXT: vmovaps %ymm4, {{[0-9]+}}(%rsp)993; AVX2-NEXT: vmovaps %ymm3, (%rsp)994; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm3995; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm4 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero996; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]997; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero998; AVX2-NEXT: vpaddq %ymm3, %ymm4, %ymm3999; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm41000; AVX2-NEXT: vpaddq %xmm4, %xmm3, %xmm31001; AVX2-NEXT: vpextrq $1, %xmm3, %rax1002; AVX2-NEXT: vmovq %xmm3, %rcx1003; AVX2-NEXT: addl %eax, %ecx1004; AVX2-NEXT: andl $7, %ecx1005; AVX2-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero1006; AVX2-NEXT: vmovlps %xmm0, (%rsp)1007; AVX2-NEXT: vmovd %xmm2, %eax1008; AVX2-NEXT: andl $1, %eax1009; AVX2-NEXT: vmovhps %xmm0, (%rsp,%rax,8)1010; AVX2-NEXT: vpextrw $1, %xmm2, %ecx1011; AVX2-NEXT: andl $1, %ecx1012; AVX2-NEXT: addq %rax, %rcx1013; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm01014; AVX2-NEXT: vmovlps %xmm0, (%rsp,%rcx,8)1015; AVX2-NEXT: vpextrw $2, %xmm2, %eax1016; AVX2-NEXT: andl $1, %eax1017; AVX2-NEXT: addq %rcx, %rax1018; AVX2-NEXT: vmovhps %xmm0, (%rsp,%rax,8)1019; AVX2-NEXT: vpextrw $3, %xmm2, %ecx1020; AVX2-NEXT: andl $1, %ecx1021; AVX2-NEXT: addq %rax, %rcx1022; AVX2-NEXT: vpextrw $4, %xmm2, %eax1023; AVX2-NEXT: andl $1, %eax1024; AVX2-NEXT: addq %rcx, %rax1025; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx1026; AVX2-NEXT: andl $7, %ecx1027; AVX2-NEXT: vmovlpd %xmm1, (%rsp,%rcx,8)1028; AVX2-NEXT: vpextrw $5, %xmm2, %ecx1029; AVX2-NEXT: andl $1, %ecx1030; AVX2-NEXT: addq %rax, %rcx1031; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax1032; AVX2-NEXT: andl $7, %eax1033; AVX2-NEXT: vmovhpd %xmm1, (%rsp,%rax,8)1034; AVX2-NEXT: vpextrw $6, %xmm2, %edx1035; AVX2-NEXT: andl $1, %edx1036; AVX2-NEXT: addq %rcx, %rdx1037; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx1038; AVX2-NEXT: andl $7, %ecx1039; AVX2-NEXT: vextractf128 $1, %ymm1, %xmm01040; AVX2-NEXT: vmovlpd %xmm0, (%rsp,%rcx,8)1041; AVX2-NEXT: vpextrw $7, %xmm2, %eax1042; AVX2-NEXT: andl $1, %eax1043; AVX2-NEXT: addq %rdx, %rax1044; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx1045; AVX2-NEXT: andl $7, %edx1046; AVX2-NEXT: vmovhpd %xmm0, (%rsp,%rdx,8)1047; AVX2-NEXT: cmpq $8, %rax1048; AVX2-NEXT: jb .LBB11_21049; AVX2-NEXT: # %bb.1:1050; AVX2-NEXT: vshufpd {{.*#+}} xmm3 = xmm0[1,0]1051; AVX2-NEXT: .LBB11_2:1052; AVX2-NEXT: cmpq $7, %rax1053; AVX2-NEXT: movl $7, %ecx1054; AVX2-NEXT: cmovbq %rax, %rcx1055; AVX2-NEXT: movl %ecx, %eax1056; AVX2-NEXT: vmovsd %xmm3, (%rsp,%rax,8)1057; AVX2-NEXT: vmovaps (%rsp), %ymm01058; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm11059; AVX2-NEXT: movq %rbp, %rsp1060; AVX2-NEXT: popq %rbp1061; AVX2-NEXT: retq1062;1063; AVX512F-LABEL: test_compress_v8f64:1064; AVX512F: # %bb.0:1065; AVX512F-NEXT: vpmovsxwq %xmm1, %zmm11066; AVX512F-NEXT: vpsllq $63, %zmm1, %zmm11067; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k11068; AVX512F-NEXT: vcompresspd %zmm0, %zmm2 {%k1}1069; AVX512F-NEXT: vmovdqa64 %zmm2, %zmm01070; AVX512F-NEXT: retq1071;1072; AVX512VL-LABEL: test_compress_v8f64:1073; AVX512VL: # %bb.0:1074; AVX512VL-NEXT: vpsllw $15, %xmm1, %xmm11075; AVX512VL-NEXT: vpmovw2m %xmm1, %k11076; AVX512VL-NEXT: vcompresspd %zmm0, %zmm2 {%k1}1077; AVX512VL-NEXT: vmovdqa64 %zmm2, %zmm01078; AVX512VL-NEXT: retq1079 %out = call <8 x double> @llvm.experimental.vector.compress(<8 x double> %vec, <8 x i1> %mask, <8 x double> %passthru)1080 ret <8 x double> %out1081}1082 1083define <16 x i8> @test_compress_v16i8(<16 x i8> %vec, <16 x i1> %mask, <16 x i8> %passthru) nounwind {1084; AVX2-LABEL: test_compress_v16i8:1085; AVX2: # %bb.0:1086; AVX2-NEXT: pushq %rbp1087; AVX2-NEXT: pushq %r151088; AVX2-NEXT: pushq %r141089; AVX2-NEXT: pushq %r131090; AVX2-NEXT: pushq %r121091; AVX2-NEXT: pushq %rbx1092; AVX2-NEXT: vpsllw $7, %xmm1, %xmm11093; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm31094; AVX2-NEXT: vpcmpgtb %xmm1, %xmm3, %xmm11095; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)1096; AVX2-NEXT: vpextrb $1, %xmm1, %r13d1097; AVX2-NEXT: vmovd %xmm1, %esi1098; AVX2-NEXT: movl %esi, %eax1099; AVX2-NEXT: andb $1, %al1100; AVX2-NEXT: subb %r13b, %al1101; AVX2-NEXT: vpextrb $2, %xmm1, %edx1102; AVX2-NEXT: subb %dl, %al1103; AVX2-NEXT: vpextrb $3, %xmm1, %ebp1104; AVX2-NEXT: subb %bpl, %al1105; AVX2-NEXT: vpextrb $4, %xmm1, %r12d1106; AVX2-NEXT: subb %r12b, %al1107; AVX2-NEXT: vpextrb $5, %xmm1, %r15d1108; AVX2-NEXT: subb %r15b, %al1109; AVX2-NEXT: vpextrb $6, %xmm1, %r14d1110; AVX2-NEXT: subb %r14b, %al1111; AVX2-NEXT: vpextrb $7, %xmm1, %ebx1112; AVX2-NEXT: subb %bl, %al1113; AVX2-NEXT: vpextrb $8, %xmm1, %r11d1114; AVX2-NEXT: subb %r11b, %al1115; AVX2-NEXT: vpextrb $9, %xmm1, %r10d1116; AVX2-NEXT: subb %r10b, %al1117; AVX2-NEXT: vpextrb $10, %xmm1, %r9d1118; AVX2-NEXT: subb %r9b, %al1119; AVX2-NEXT: vpextrb $11, %xmm1, %r8d1120; AVX2-NEXT: subb %r8b, %al1121; AVX2-NEXT: vpextrb $12, %xmm1, %edi1122; AVX2-NEXT: subb %dil, %al1123; AVX2-NEXT: vpextrb $13, %xmm1, %ecx1124; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1125; AVX2-NEXT: subb %cl, %al1126; AVX2-NEXT: vpextrb $14, %xmm1, %ecx1127; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1128; AVX2-NEXT: subb %cl, %al1129; AVX2-NEXT: vpextrb $15, %xmm1, %ecx1130; AVX2-NEXT: subb %cl, %al1131; AVX2-NEXT: movzbl %al, %eax1132; AVX2-NEXT: andl $15, %eax1133; AVX2-NEXT: movzbl -40(%rsp,%rax), %eax1134; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill1135; AVX2-NEXT: vpextrb $0, %xmm0, -{{[0-9]+}}(%rsp)1136; AVX2-NEXT: andl $1, %esi1137; AVX2-NEXT: vpextrb $1, %xmm0, -40(%rsp,%rsi)1138; AVX2-NEXT: andl $1, %r13d1139; AVX2-NEXT: addq %rsi, %r131140; AVX2-NEXT: vpextrb $2, %xmm0, -40(%rsp,%r13)1141; AVX2-NEXT: andl $1, %edx1142; AVX2-NEXT: addq %r13, %rdx1143; AVX2-NEXT: vpextrb $3, %xmm0, -40(%rsp,%rdx)1144; AVX2-NEXT: andl $1, %ebp1145; AVX2-NEXT: addq %rdx, %rbp1146; AVX2-NEXT: vpextrb $4, %xmm0, -40(%rsp,%rbp)1147; AVX2-NEXT: andl $1, %r12d1148; AVX2-NEXT: addq %rbp, %r121149; AVX2-NEXT: andl $1, %r15d1150; AVX2-NEXT: addq %r12, %r151151; AVX2-NEXT: # kill: def $r12d killed $r12d killed $r12 def $r121152; AVX2-NEXT: andl $15, %r12d1153; AVX2-NEXT: vpextrb $5, %xmm0, -40(%rsp,%r12)1154; AVX2-NEXT: andl $1, %r14d1155; AVX2-NEXT: addq %r15, %r141156; AVX2-NEXT: # kill: def $r15d killed $r15d killed $r15 def $r151157; AVX2-NEXT: andl $15, %r15d1158; AVX2-NEXT: vpextrb $6, %xmm0, -40(%rsp,%r15)1159; AVX2-NEXT: andl $1, %ebx1160; AVX2-NEXT: addq %r14, %rbx1161; AVX2-NEXT: # kill: def $r14d killed $r14d killed $r14 def $r141162; AVX2-NEXT: andl $15, %r14d1163; AVX2-NEXT: vpextrb $7, %xmm0, -40(%rsp,%r14)1164; AVX2-NEXT: andl $1, %r11d1165; AVX2-NEXT: addq %rbx, %r111166; AVX2-NEXT: # kill: def $ebx killed $ebx killed $rbx def $rbx1167; AVX2-NEXT: andl $15, %ebx1168; AVX2-NEXT: vpextrb $8, %xmm0, -40(%rsp,%rbx)1169; AVX2-NEXT: andl $1, %r10d1170; AVX2-NEXT: addq %r11, %r101171; AVX2-NEXT: # kill: def $r11d killed $r11d killed $r11 def $r111172; AVX2-NEXT: andl $15, %r11d1173; AVX2-NEXT: vpextrb $9, %xmm0, -40(%rsp,%r11)1174; AVX2-NEXT: andl $1, %r9d1175; AVX2-NEXT: addq %r10, %r91176; AVX2-NEXT: # kill: def $r10d killed $r10d killed $r10 def $r101177; AVX2-NEXT: andl $15, %r10d1178; AVX2-NEXT: vpextrb $10, %xmm0, -40(%rsp,%r10)1179; AVX2-NEXT: andl $1, %r8d1180; AVX2-NEXT: addq %r9, %r81181; AVX2-NEXT: # kill: def $r9d killed $r9d killed $r9 def $r91182; AVX2-NEXT: andl $15, %r9d1183; AVX2-NEXT: vpextrb $11, %xmm0, -40(%rsp,%r9)1184; AVX2-NEXT: andl $1, %edi1185; AVX2-NEXT: addq %r8, %rdi1186; AVX2-NEXT: # kill: def $r8d killed $r8d killed $r8 def $r81187; AVX2-NEXT: andl $15, %r8d1188; AVX2-NEXT: vpextrb $12, %xmm0, -40(%rsp,%r8)1189; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload1190; AVX2-NEXT: andl $1, %esi1191; AVX2-NEXT: addq %rdi, %rsi1192; AVX2-NEXT: # kill: def $edi killed $edi killed $rdi def $rdi1193; AVX2-NEXT: andl $15, %edi1194; AVX2-NEXT: vpextrb $13, %xmm0, -40(%rsp,%rdi)1195; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload1196; AVX2-NEXT: andl $1, %eax1197; AVX2-NEXT: addq %rsi, %rax1198; AVX2-NEXT: # kill: def $esi killed $esi killed $rsi def $rsi1199; AVX2-NEXT: andl $15, %esi1200; AVX2-NEXT: vpextrb $14, %xmm0, -40(%rsp,%rsi)1201; AVX2-NEXT: andl $1, %ecx1202; AVX2-NEXT: addq %rax, %rcx1203; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax1204; AVX2-NEXT: andl $15, %eax1205; AVX2-NEXT: vpextrb $15, %xmm0, -40(%rsp,%rax)1206; AVX2-NEXT: cmpq $15, %rcx1207; AVX2-NEXT: movl $15, %eax1208; AVX2-NEXT: cmovbq %rcx, %rax1209; AVX2-NEXT: vpextrb $15, %xmm0, %ecx1210; AVX2-NEXT: cmovbel {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Folded Reload1211; AVX2-NEXT: movb %cl, -40(%rsp,%rax)1212; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm01213; AVX2-NEXT: popq %rbx1214; AVX2-NEXT: popq %r121215; AVX2-NEXT: popq %r131216; AVX2-NEXT: popq %r141217; AVX2-NEXT: popq %r151218; AVX2-NEXT: popq %rbp1219; AVX2-NEXT: retq1220;1221; AVX512F-LABEL: test_compress_v16i8:1222; AVX512F: # %bb.0:1223; AVX512F-NEXT: vpmovsxbd %xmm1, %zmm11224; AVX512F-NEXT: vpslld $31, %zmm1, %zmm11225; AVX512F-NEXT: vptestmd %zmm1, %zmm1, %k11226; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero1227; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero1228; AVX512F-NEXT: vpcompressd %zmm0, %zmm1 {%k1}1229; AVX512F-NEXT: vpmovdb %zmm1, %xmm01230; AVX512F-NEXT: vzeroupper1231; AVX512F-NEXT: retq1232;1233; AVX512VL-LABEL: test_compress_v16i8:1234; AVX512VL: # %bb.0:1235; AVX512VL-NEXT: vpsllw $7, %xmm1, %xmm11236; AVX512VL-NEXT: vpmovb2m %xmm1, %k11237; AVX512VL-NEXT: vpcompressb %xmm0, %xmm2 {%k1}1238; AVX512VL-NEXT: vmovdqa %xmm2, %xmm01239; AVX512VL-NEXT: retq1240 %out = call <16 x i8> @llvm.experimental.vector.compress(<16 x i8> %vec, <16 x i1> %mask, <16 x i8> %passthru)1241 ret <16 x i8> %out1242}1243 1244define <8 x i16> @test_compress_v8i16(<8 x i16> %vec, <8 x i1> %mask, <8 x i16> %passthru) nounwind {1245; AVX2-LABEL: test_compress_v8i16:1246; AVX2: # %bb.0:1247; AVX2-NEXT: pushq %rbx1248; AVX2-NEXT: vpsllw $15, %xmm1, %xmm11249; AVX2-NEXT: vpsraw $15, %xmm1, %xmm11250; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)1251; AVX2-NEXT: vpextrw $1, %xmm1, %eax1252; AVX2-NEXT: andl $1, %eax1253; AVX2-NEXT: vmovd %xmm1, %ecx1254; AVX2-NEXT: andl $1, %ecx1255; AVX2-NEXT: leal (%rcx,%rax), %esi1256; AVX2-NEXT: vpextrw $2, %xmm1, %edi1257; AVX2-NEXT: andl $1, %edi1258; AVX2-NEXT: vpextrw $3, %xmm1, %edx1259; AVX2-NEXT: andl $1, %edx1260; AVX2-NEXT: leal (%rdi,%rdx), %r10d1261; AVX2-NEXT: addl %esi, %r10d1262; AVX2-NEXT: vpextrw $4, %xmm1, %r9d1263; AVX2-NEXT: andl $1, %r9d1264; AVX2-NEXT: vpextrw $5, %xmm1, %esi1265; AVX2-NEXT: andl $1, %esi1266; AVX2-NEXT: leal (%r9,%rsi), %r11d1267; AVX2-NEXT: vpextrw $6, %xmm1, %r8d1268; AVX2-NEXT: andl $1, %r8d1269; AVX2-NEXT: addl %r8d, %r11d1270; AVX2-NEXT: addl %r10d, %r11d1271; AVX2-NEXT: vpextrw $7, %xmm1, %r10d1272; AVX2-NEXT: andl $1, %r10d1273; AVX2-NEXT: addl %r10d, %r11d1274; AVX2-NEXT: andl $7, %r11d1275; AVX2-NEXT: addq %rcx, %rax1276; AVX2-NEXT: addq %rax, %rdi1277; AVX2-NEXT: addq %rdi, %rdx1278; AVX2-NEXT: addq %rdx, %r91279; AVX2-NEXT: addq %r9, %rsi1280; AVX2-NEXT: addq %rsi, %r81281; AVX2-NEXT: addq %r8, %r101282; AVX2-NEXT: vpextrw $7, %xmm0, %ebx1283; AVX2-NEXT: cmpq $8, %r101284; AVX2-NEXT: cmovbw -16(%rsp,%r11,2), %bx1285; AVX2-NEXT: vpextrw $0, %xmm0, -{{[0-9]+}}(%rsp)1286; AVX2-NEXT: vpextrw $1, %xmm0, -16(%rsp,%rcx,2)1287; AVX2-NEXT: vpextrw $2, %xmm0, -16(%rsp,%rax,2)1288; AVX2-NEXT: vpextrw $3, %xmm0, -16(%rsp,%rdi,2)1289; AVX2-NEXT: andl $7, %edx1290; AVX2-NEXT: vpextrw $4, %xmm0, -16(%rsp,%rdx,2)1291; AVX2-NEXT: andl $7, %r9d1292; AVX2-NEXT: vpextrw $5, %xmm0, -16(%rsp,%r9,2)1293; AVX2-NEXT: andl $7, %esi1294; AVX2-NEXT: vpextrw $6, %xmm0, -16(%rsp,%rsi,2)1295; AVX2-NEXT: andl $7, %r8d1296; AVX2-NEXT: vpextrw $7, %xmm0, -16(%rsp,%r8,2)1297; AVX2-NEXT: cmpq $7, %r101298; AVX2-NEXT: movl $7, %eax1299; AVX2-NEXT: cmovbq %r10, %rax1300; AVX2-NEXT: movl %eax, %eax1301; AVX2-NEXT: movw %bx, -16(%rsp,%rax,2)1302; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm01303; AVX2-NEXT: popq %rbx1304; AVX2-NEXT: retq1305;1306; AVX512F-LABEL: test_compress_v8i16:1307; AVX512F: # %bb.0:1308; AVX512F-NEXT: vpmovsxwq %xmm1, %zmm11309; AVX512F-NEXT: vpsllq $63, %zmm1, %zmm11310; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k11311; AVX512F-NEXT: vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero1312; AVX512F-NEXT: vpmovzxwq {{.*#+}} zmm1 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero1313; AVX512F-NEXT: vpcompressq %zmm0, %zmm1 {%k1}1314; AVX512F-NEXT: vpmovqw %zmm1, %xmm01315; AVX512F-NEXT: vzeroupper1316; AVX512F-NEXT: retq1317;1318; AVX512VL-LABEL: test_compress_v8i16:1319; AVX512VL: # %bb.0:1320; AVX512VL-NEXT: vpsllw $15, %xmm1, %xmm11321; AVX512VL-NEXT: vpmovw2m %xmm1, %k11322; AVX512VL-NEXT: vpcompressw %xmm0, %xmm2 {%k1}1323; AVX512VL-NEXT: vmovdqa %xmm2, %xmm01324; AVX512VL-NEXT: retq1325 %out = call <8 x i16> @llvm.experimental.vector.compress(<8 x i16> %vec, <8 x i1> %mask, <8 x i16> %passthru)1326 ret <8 x i16> %out1327}1328 1329define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8> %passthru) nounwind {1330; AVX2-LABEL: test_compress_v32i8:1331; AVX2: # %bb.0:1332; AVX2-NEXT: pushq %rbp1333; AVX2-NEXT: movq %rsp, %rbp1334; AVX2-NEXT: andq $-32, %rsp1335; AVX2-NEXT: subq $64, %rsp1336; AVX2-NEXT: vpsllw $7, %ymm1, %ymm11337; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm31338; AVX2-NEXT: vpcmpgtb %ymm1, %ymm3, %ymm31339; AVX2-NEXT: vmovaps %ymm2, (%rsp)1340; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm11341; AVX2-NEXT: vpbroadcastb {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]1342; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm41343; AVX2-NEXT: vpand %xmm2, %xmm3, %xmm21344; AVX2-NEXT: vpaddb %xmm4, %xmm2, %xmm21345; AVX2-NEXT: vpextrb $1, %xmm2, %eax1346; AVX2-NEXT: vmovd %xmm2, %ecx1347; AVX2-NEXT: addb %al, %cl1348; AVX2-NEXT: vpextrb $2, %xmm2, %eax1349; AVX2-NEXT: vpextrb $3, %xmm2, %edx1350; AVX2-NEXT: addb %al, %dl1351; AVX2-NEXT: addb %cl, %dl1352; AVX2-NEXT: vpextrb $4, %xmm2, %eax1353; AVX2-NEXT: vpextrb $5, %xmm2, %ecx1354; AVX2-NEXT: addb %al, %cl1355; AVX2-NEXT: vpextrb $6, %xmm2, %eax1356; AVX2-NEXT: addb %cl, %al1357; AVX2-NEXT: addb %dl, %al1358; AVX2-NEXT: vpextrb $7, %xmm2, %ecx1359; AVX2-NEXT: vpextrb $8, %xmm2, %edx1360; AVX2-NEXT: addb %cl, %dl1361; AVX2-NEXT: vpextrb $9, %xmm2, %ecx1362; AVX2-NEXT: addb %dl, %cl1363; AVX2-NEXT: vpextrb $10, %xmm2, %edx1364; AVX2-NEXT: addb %cl, %dl1365; AVX2-NEXT: addb %al, %dl1366; AVX2-NEXT: vpextrb $11, %xmm2, %eax1367; AVX2-NEXT: vpextrb $12, %xmm2, %ecx1368; AVX2-NEXT: addb %al, %cl1369; AVX2-NEXT: vpextrb $13, %xmm2, %eax1370; AVX2-NEXT: addb %cl, %al1371; AVX2-NEXT: vpextrb $14, %xmm2, %ecx1372; AVX2-NEXT: addb %al, %cl1373; AVX2-NEXT: vpextrb $15, %xmm2, %eax1374; AVX2-NEXT: addb %cl, %al1375; AVX2-NEXT: addb %dl, %al1376; AVX2-NEXT: movzbl %al, %eax1377; AVX2-NEXT: andl $31, %eax1378; AVX2-NEXT: movzbl (%rsp,%rax), %eax1379; AVX2-NEXT: vpextrb $0, %xmm0, (%rsp)1380; AVX2-NEXT: vmovd %xmm3, %ecx1381; AVX2-NEXT: andl $1, %ecx1382; AVX2-NEXT: vpextrb $1, %xmm0, (%rsp,%rcx)1383; AVX2-NEXT: vpextrb $1, %xmm3, %edx1384; AVX2-NEXT: andl $1, %edx1385; AVX2-NEXT: addq %rcx, %rdx1386; AVX2-NEXT: vpextrb $2, %xmm0, (%rsp,%rdx)1387; AVX2-NEXT: vpextrb $2, %xmm3, %ecx1388; AVX2-NEXT: andl $1, %ecx1389; AVX2-NEXT: addq %rdx, %rcx1390; AVX2-NEXT: vpextrb $3, %xmm0, (%rsp,%rcx)1391; AVX2-NEXT: vpextrb $3, %xmm3, %edx1392; AVX2-NEXT: andl $1, %edx1393; AVX2-NEXT: addq %rcx, %rdx1394; AVX2-NEXT: vpextrb $4, %xmm0, (%rsp,%rdx)1395; AVX2-NEXT: vpextrb $4, %xmm3, %ecx1396; AVX2-NEXT: andl $1, %ecx1397; AVX2-NEXT: addq %rdx, %rcx1398; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rcx)1399; AVX2-NEXT: vpextrb $5, %xmm3, %edx1400; AVX2-NEXT: andl $1, %edx1401; AVX2-NEXT: addq %rcx, %rdx1402; AVX2-NEXT: vpextrb $6, %xmm3, %ecx1403; AVX2-NEXT: andl $1, %ecx1404; AVX2-NEXT: addq %rdx, %rcx1405; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx1406; AVX2-NEXT: andl $31, %edx1407; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%rdx)1408; AVX2-NEXT: vpextrb $7, %xmm3, %edx1409; AVX2-NEXT: andl $1, %edx1410; AVX2-NEXT: addq %rcx, %rdx1411; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx1412; AVX2-NEXT: andl $31, %ecx1413; AVX2-NEXT: vpextrb $7, %xmm0, (%rsp,%rcx)1414; AVX2-NEXT: vpextrb $8, %xmm3, %ecx1415; AVX2-NEXT: andl $1, %ecx1416; AVX2-NEXT: addq %rdx, %rcx1417; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx1418; AVX2-NEXT: andl $31, %edx1419; AVX2-NEXT: vpextrb $8, %xmm0, (%rsp,%rdx)1420; AVX2-NEXT: vpextrb $9, %xmm3, %edx1421; AVX2-NEXT: andl $1, %edx1422; AVX2-NEXT: addq %rcx, %rdx1423; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx1424; AVX2-NEXT: andl $31, %ecx1425; AVX2-NEXT: vpextrb $9, %xmm0, (%rsp,%rcx)1426; AVX2-NEXT: vpextrb $10, %xmm3, %ecx1427; AVX2-NEXT: andl $1, %ecx1428; AVX2-NEXT: addq %rdx, %rcx1429; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx1430; AVX2-NEXT: andl $31, %edx1431; AVX2-NEXT: vpextrb $10, %xmm0, (%rsp,%rdx)1432; AVX2-NEXT: vpextrb $11, %xmm3, %edx1433; AVX2-NEXT: andl $1, %edx1434; AVX2-NEXT: addq %rcx, %rdx1435; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx1436; AVX2-NEXT: andl $31, %ecx1437; AVX2-NEXT: vpextrb $11, %xmm0, (%rsp,%rcx)1438; AVX2-NEXT: vpextrb $12, %xmm3, %ecx1439; AVX2-NEXT: andl $1, %ecx1440; AVX2-NEXT: addq %rdx, %rcx1441; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx1442; AVX2-NEXT: andl $31, %edx1443; AVX2-NEXT: vpextrb $12, %xmm0, (%rsp,%rdx)1444; AVX2-NEXT: vpextrb $13, %xmm3, %edx1445; AVX2-NEXT: andl $1, %edx1446; AVX2-NEXT: addq %rcx, %rdx1447; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx1448; AVX2-NEXT: andl $31, %ecx1449; AVX2-NEXT: vpextrb $13, %xmm0, (%rsp,%rcx)1450; AVX2-NEXT: vpextrb $14, %xmm3, %ecx1451; AVX2-NEXT: andl $1, %ecx1452; AVX2-NEXT: addq %rdx, %rcx1453; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx1454; AVX2-NEXT: andl $31, %edx1455; AVX2-NEXT: vpextrb $14, %xmm0, (%rsp,%rdx)1456; AVX2-NEXT: vpextrb $15, %xmm3, %edx1457; AVX2-NEXT: andl $1, %edx1458; AVX2-NEXT: addq %rcx, %rdx1459; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx1460; AVX2-NEXT: andl $31, %ecx1461; AVX2-NEXT: vpextrb $15, %xmm0, (%rsp,%rcx)1462; AVX2-NEXT: vmovd %xmm1, %ecx1463; AVX2-NEXT: andl $1, %ecx1464; AVX2-NEXT: addq %rdx, %rcx1465; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx1466; AVX2-NEXT: andl $31, %edx1467; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm01468; AVX2-NEXT: vpextrb $0, %xmm0, (%rsp,%rdx)1469; AVX2-NEXT: vpextrb $1, %xmm1, %edx1470; AVX2-NEXT: andl $1, %edx1471; AVX2-NEXT: addq %rcx, %rdx1472; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx1473; AVX2-NEXT: andl $31, %ecx1474; AVX2-NEXT: vpextrb $1, %xmm0, (%rsp,%rcx)1475; AVX2-NEXT: vpextrb $2, %xmm1, %ecx1476; AVX2-NEXT: andl $1, %ecx1477; AVX2-NEXT: addq %rdx, %rcx1478; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx1479; AVX2-NEXT: andl $31, %edx1480; AVX2-NEXT: vpextrb $2, %xmm0, (%rsp,%rdx)1481; AVX2-NEXT: vpextrb $3, %xmm1, %edx1482; AVX2-NEXT: andl $1, %edx1483; AVX2-NEXT: addq %rcx, %rdx1484; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx1485; AVX2-NEXT: andl $31, %ecx1486; AVX2-NEXT: vpextrb $3, %xmm0, (%rsp,%rcx)1487; AVX2-NEXT: vpextrb $4, %xmm1, %ecx1488; AVX2-NEXT: andl $1, %ecx1489; AVX2-NEXT: addq %rdx, %rcx1490; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx1491; AVX2-NEXT: andl $31, %edx1492; AVX2-NEXT: vpextrb $4, %xmm0, (%rsp,%rdx)1493; AVX2-NEXT: vpextrb $5, %xmm1, %edx1494; AVX2-NEXT: andl $1, %edx1495; AVX2-NEXT: addq %rcx, %rdx1496; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx1497; AVX2-NEXT: andl $31, %ecx1498; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rcx)1499; AVX2-NEXT: vpextrb $6, %xmm1, %ecx1500; AVX2-NEXT: andl $1, %ecx1501; AVX2-NEXT: addq %rdx, %rcx1502; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx1503; AVX2-NEXT: andl $31, %edx1504; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%rdx)1505; AVX2-NEXT: vpextrb $7, %xmm1, %edx1506; AVX2-NEXT: andl $1, %edx1507; AVX2-NEXT: addq %rcx, %rdx1508; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx1509; AVX2-NEXT: andl $31, %ecx1510; AVX2-NEXT: vpextrb $7, %xmm0, (%rsp,%rcx)1511; AVX2-NEXT: vpextrb $8, %xmm1, %ecx1512; AVX2-NEXT: andl $1, %ecx1513; AVX2-NEXT: addq %rdx, %rcx1514; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx1515; AVX2-NEXT: andl $31, %edx1516; AVX2-NEXT: vpextrb $8, %xmm0, (%rsp,%rdx)1517; AVX2-NEXT: vpextrb $9, %xmm1, %edx1518; AVX2-NEXT: andl $1, %edx1519; AVX2-NEXT: addq %rcx, %rdx1520; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx1521; AVX2-NEXT: andl $31, %ecx1522; AVX2-NEXT: vpextrb $9, %xmm0, (%rsp,%rcx)1523; AVX2-NEXT: vpextrb $10, %xmm1, %ecx1524; AVX2-NEXT: andl $1, %ecx1525; AVX2-NEXT: addq %rdx, %rcx1526; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx1527; AVX2-NEXT: andl $31, %edx1528; AVX2-NEXT: vpextrb $10, %xmm0, (%rsp,%rdx)1529; AVX2-NEXT: vpextrb $11, %xmm1, %edx1530; AVX2-NEXT: andl $1, %edx1531; AVX2-NEXT: addq %rcx, %rdx1532; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx1533; AVX2-NEXT: andl $31, %ecx1534; AVX2-NEXT: vpextrb $11, %xmm0, (%rsp,%rcx)1535; AVX2-NEXT: vpextrb $12, %xmm1, %ecx1536; AVX2-NEXT: andl $1, %ecx1537; AVX2-NEXT: addq %rdx, %rcx1538; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx1539; AVX2-NEXT: andl $31, %edx1540; AVX2-NEXT: vpextrb $12, %xmm0, (%rsp,%rdx)1541; AVX2-NEXT: vpextrb $13, %xmm1, %edx1542; AVX2-NEXT: andl $1, %edx1543; AVX2-NEXT: addq %rcx, %rdx1544; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx1545; AVX2-NEXT: andl $31, %ecx1546; AVX2-NEXT: vpextrb $13, %xmm0, (%rsp,%rcx)1547; AVX2-NEXT: vpextrb $14, %xmm1, %ecx1548; AVX2-NEXT: andl $1, %ecx1549; AVX2-NEXT: addq %rdx, %rcx1550; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx1551; AVX2-NEXT: andl $31, %edx1552; AVX2-NEXT: vpextrb $14, %xmm0, (%rsp,%rdx)1553; AVX2-NEXT: vpextrb $15, %xmm1, %edx1554; AVX2-NEXT: andl $1, %edx1555; AVX2-NEXT: addq %rcx, %rdx1556; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx1557; AVX2-NEXT: andl $31, %ecx1558; AVX2-NEXT: vpextrb $15, %xmm0, (%rsp,%rcx)1559; AVX2-NEXT: cmpq $31, %rdx1560; AVX2-NEXT: movl $31, %ecx1561; AVX2-NEXT: cmovbq %rdx, %rcx1562; AVX2-NEXT: vpextrb $15, %xmm0, %edx1563; AVX2-NEXT: cmovbel %eax, %edx1564; AVX2-NEXT: movb %dl, (%rsp,%rcx)1565; AVX2-NEXT: vmovaps (%rsp), %ymm01566; AVX2-NEXT: movq %rbp, %rsp1567; AVX2-NEXT: popq %rbp1568; AVX2-NEXT: retq1569;1570; AVX512F-LABEL: test_compress_v32i8:1571; AVX512F: # %bb.0:1572; AVX512F-NEXT: pushq %rbp1573; AVX512F-NEXT: movq %rsp, %rbp1574; AVX512F-NEXT: andq $-32, %rsp1575; AVX512F-NEXT: subq $64, %rsp1576; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm31577; AVX512F-NEXT: vpmovsxbd %xmm3, %zmm31578; AVX512F-NEXT: vpslld $31, %zmm3, %zmm31579; AVX512F-NEXT: vptestmd %zmm3, %zmm3, %k11580; AVX512F-NEXT: vpmovsxbd %xmm1, %zmm31581; AVX512F-NEXT: vpslld $31, %zmm3, %zmm31582; AVX512F-NEXT: vptestmd %zmm3, %zmm3, %k21583; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero1584; AVX512F-NEXT: vpcompressd %zmm3, %zmm3 {%k2} {z}1585; AVX512F-NEXT: vpmovdb %zmm3, (%rsp)1586; AVX512F-NEXT: kshiftrw $8, %k2, %k01587; AVX512F-NEXT: kxorw %k0, %k2, %k01588; AVX512F-NEXT: kshiftrw $4, %k0, %k21589; AVX512F-NEXT: kxorw %k2, %k0, %k01590; AVX512F-NEXT: kshiftrw $2, %k0, %k21591; AVX512F-NEXT: kxorw %k2, %k0, %k01592; AVX512F-NEXT: kshiftrw $1, %k0, %k21593; AVX512F-NEXT: kxorw %k2, %k0, %k01594; AVX512F-NEXT: kmovw %k0, %eax1595; AVX512F-NEXT: andl $31, %eax1596; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm01597; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero1598; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}1599; AVX512F-NEXT: vpmovdb %zmm0, (%rsp,%rax)1600; AVX512F-NEXT: vpsllw $7, %ymm1, %ymm01601; AVX512F-NEXT: vpblendvb %ymm0, (%rsp), %ymm2, %ymm01602; AVX512F-NEXT: movq %rbp, %rsp1603; AVX512F-NEXT: popq %rbp1604; AVX512F-NEXT: retq1605;1606; AVX512VL-LABEL: test_compress_v32i8:1607; AVX512VL: # %bb.0:1608; AVX512VL-NEXT: vpsllw $7, %ymm1, %ymm11609; AVX512VL-NEXT: vpmovb2m %ymm1, %k11610; AVX512VL-NEXT: vpcompressb %ymm0, %ymm2 {%k1}1611; AVX512VL-NEXT: vmovdqa %ymm2, %ymm01612; AVX512VL-NEXT: retq1613 %out = call <32 x i8> @llvm.experimental.vector.compress(<32 x i8> %vec, <32 x i1> %mask, <32 x i8> %passthru)1614 ret <32 x i8> %out1615}1616 1617define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x i16> %passthru) nounwind {1618; AVX2-LABEL: test_compress_v16i16:1619; AVX2: # %bb.0:1620; AVX2-NEXT: pushq %rbp1621; AVX2-NEXT: movq %rsp, %rbp1622; AVX2-NEXT: pushq %r151623; AVX2-NEXT: pushq %r141624; AVX2-NEXT: pushq %r131625; AVX2-NEXT: pushq %r121626; AVX2-NEXT: pushq %rbx1627; AVX2-NEXT: andq $-32, %rsp1628; AVX2-NEXT: subq $96, %rsp1629; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero1630; AVX2-NEXT: vpsllw $15, %ymm1, %ymm31631; AVX2-NEXT: vpsraw $15, %ymm3, %ymm11632; AVX2-NEXT: vmovaps %ymm2, (%rsp)1633; AVX2-NEXT: vpsrlw $15, %ymm3, %ymm21634; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm31635; AVX2-NEXT: vpaddw %xmm3, %xmm2, %xmm21636; AVX2-NEXT: vpextrw $1, %xmm2, %eax1637; AVX2-NEXT: vmovd %xmm2, %ecx1638; AVX2-NEXT: addl %eax, %ecx1639; AVX2-NEXT: vpextrw $2, %xmm2, %eax1640; AVX2-NEXT: vpextrw $3, %xmm2, %edx1641; AVX2-NEXT: addl %eax, %edx1642; AVX2-NEXT: addl %ecx, %edx1643; AVX2-NEXT: vpextrw $4, %xmm2, %eax1644; AVX2-NEXT: vpextrw $5, %xmm2, %ecx1645; AVX2-NEXT: addl %eax, %ecx1646; AVX2-NEXT: vpextrw $6, %xmm2, %eax1647; AVX2-NEXT: addl %ecx, %eax1648; AVX2-NEXT: addl %edx, %eax1649; AVX2-NEXT: vpextrw $7, %xmm2, %ecx1650; AVX2-NEXT: addl %eax, %ecx1651; AVX2-NEXT: andl $15, %ecx1652; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1653; AVX2-NEXT: vpextrw $1, %xmm1, %eax1654; AVX2-NEXT: andl $1, %eax1655; AVX2-NEXT: vmovd %xmm1, %ecx1656; AVX2-NEXT: andl $1, %ecx1657; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1658; AVX2-NEXT: addq %rcx, %rax1659; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1660; AVX2-NEXT: vpextrw $2, %xmm1, %ecx1661; AVX2-NEXT: andl $1, %ecx1662; AVX2-NEXT: addq %rax, %rcx1663; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1664; AVX2-NEXT: vpextrw $3, %xmm1, %eax1665; AVX2-NEXT: andl $1, %eax1666; AVX2-NEXT: addq %rcx, %rax1667; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1668; AVX2-NEXT: vpextrw $4, %xmm1, %r8d1669; AVX2-NEXT: andl $1, %r8d1670; AVX2-NEXT: addq %rax, %r81671; AVX2-NEXT: vpextrw $5, %xmm1, %r9d1672; AVX2-NEXT: andl $1, %r9d1673; AVX2-NEXT: addq %r8, %r91674; AVX2-NEXT: vpextrw $6, %xmm1, %r10d1675; AVX2-NEXT: andl $1, %r10d1676; AVX2-NEXT: addq %r9, %r101677; AVX2-NEXT: vpextrw $7, %xmm1, %r11d1678; AVX2-NEXT: andl $1, %r11d1679; AVX2-NEXT: addq %r10, %r111680; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm11681; AVX2-NEXT: vmovd %xmm1, %ebx1682; AVX2-NEXT: andl $1, %ebx1683; AVX2-NEXT: addq %r11, %rbx1684; AVX2-NEXT: vpextrw $1, %xmm1, %r14d1685; AVX2-NEXT: andl $1, %r14d1686; AVX2-NEXT: addq %rbx, %r141687; AVX2-NEXT: vpextrw $2, %xmm1, %r15d1688; AVX2-NEXT: andl $1, %r15d1689; AVX2-NEXT: addq %r14, %r151690; AVX2-NEXT: vpextrw $3, %xmm1, %r12d1691; AVX2-NEXT: andl $1, %r12d1692; AVX2-NEXT: addq %r15, %r121693; AVX2-NEXT: vpextrw $4, %xmm1, %r13d1694; AVX2-NEXT: andl $1, %r13d1695; AVX2-NEXT: addq %r12, %r131696; AVX2-NEXT: vpextrw $5, %xmm1, %edx1697; AVX2-NEXT: andl $1, %edx1698; AVX2-NEXT: addq %r13, %rdx1699; AVX2-NEXT: vpextrw $6, %xmm1, %ecx1700; AVX2-NEXT: andl $1, %ecx1701; AVX2-NEXT: addq %rdx, %rcx1702; AVX2-NEXT: vpextrw $7, %xmm1, %edi1703; AVX2-NEXT: andl $1, %edi1704; AVX2-NEXT: addq %rcx, %rdi1705; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm11706; AVX2-NEXT: cmpq $16, %rdi1707; AVX2-NEXT: vpextrw $7, %xmm1, %eax1708; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload1709; AVX2-NEXT: cmovbw (%rsp,%rsi,2), %ax1710; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill1711; AVX2-NEXT: vpextrw $0, %xmm0, (%rsp)1712; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload1713; AVX2-NEXT: vpextrw $1, %xmm0, (%rsp,%rsi,2)1714; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload1715; AVX2-NEXT: vpextrw $2, %xmm0, (%rsp,%rsi,2)1716; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload1717; AVX2-NEXT: vpextrw $3, %xmm0, (%rsp,%rsi,2)1718; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload1719; AVX2-NEXT: vpextrw $4, %xmm0, (%rsp,%rax,2)1720; AVX2-NEXT: andl $15, %r8d1721; AVX2-NEXT: vpextrw $5, %xmm0, (%rsp,%r8,2)1722; AVX2-NEXT: andl $15, %r9d1723; AVX2-NEXT: vpextrw $6, %xmm0, (%rsp,%r9,2)1724; AVX2-NEXT: andl $15, %r10d1725; AVX2-NEXT: vpextrw $7, %xmm0, (%rsp,%r10,2)1726; AVX2-NEXT: andl $15, %r11d1727; AVX2-NEXT: vpextrw $0, %xmm1, (%rsp,%r11,2)1728; AVX2-NEXT: andl $15, %ebx1729; AVX2-NEXT: vpextrw $1, %xmm1, (%rsp,%rbx,2)1730; AVX2-NEXT: andl $15, %r14d1731; AVX2-NEXT: vpextrw $2, %xmm1, (%rsp,%r14,2)1732; AVX2-NEXT: andl $15, %r15d1733; AVX2-NEXT: vpextrw $3, %xmm1, (%rsp,%r15,2)1734; AVX2-NEXT: andl $15, %r12d1735; AVX2-NEXT: vpextrw $4, %xmm1, (%rsp,%r12,2)1736; AVX2-NEXT: andl $15, %r13d1737; AVX2-NEXT: vpextrw $5, %xmm1, (%rsp,%r13,2)1738; AVX2-NEXT: andl $15, %edx1739; AVX2-NEXT: vpextrw $6, %xmm1, (%rsp,%rdx,2)1740; AVX2-NEXT: andl $15, %ecx1741; AVX2-NEXT: vpextrw $7, %xmm1, (%rsp,%rcx,2)1742; AVX2-NEXT: cmpq $15, %rdi1743; AVX2-NEXT: movl $15, %eax1744; AVX2-NEXT: cmovbq %rdi, %rax1745; AVX2-NEXT: movl %eax, %eax1746; AVX2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload1747; AVX2-NEXT: movw %cx, (%rsp,%rax,2)1748; AVX2-NEXT: vmovaps (%rsp), %ymm01749; AVX2-NEXT: leaq -40(%rbp), %rsp1750; AVX2-NEXT: popq %rbx1751; AVX2-NEXT: popq %r121752; AVX2-NEXT: popq %r131753; AVX2-NEXT: popq %r141754; AVX2-NEXT: popq %r151755; AVX2-NEXT: popq %rbp1756; AVX2-NEXT: retq1757;1758; AVX512F-LABEL: test_compress_v16i16:1759; AVX512F: # %bb.0:1760; AVX512F-NEXT: vpmovsxbd %xmm1, %zmm11761; AVX512F-NEXT: vpslld $31, %zmm1, %zmm11762; AVX512F-NEXT: vptestmd %zmm1, %zmm1, %k11763; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero1764; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero1765; AVX512F-NEXT: vpcompressd %zmm0, %zmm1 {%k1}1766; AVX512F-NEXT: vpmovdw %zmm1, %ymm01767; AVX512F-NEXT: retq1768;1769; AVX512VL-LABEL: test_compress_v16i16:1770; AVX512VL: # %bb.0:1771; AVX512VL-NEXT: vpsllw $7, %xmm1, %xmm11772; AVX512VL-NEXT: vpmovb2m %xmm1, %k11773; AVX512VL-NEXT: vpcompressw %ymm0, %ymm2 {%k1}1774; AVX512VL-NEXT: vmovdqa %ymm2, %ymm01775; AVX512VL-NEXT: retq1776 %out = call <16 x i16> @llvm.experimental.vector.compress(<16 x i16> %vec, <16 x i1> %mask, <16 x i16> %passthru)1777 ret <16 x i16> %out1778}1779 1780define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8> %passthru) nounwind {1781; AVX2-LABEL: test_compress_v64i8:1782; AVX2: # %bb.0:1783; AVX2-NEXT: pushq %rbp1784; AVX2-NEXT: movq %rsp, %rbp1785; AVX2-NEXT: pushq %r151786; AVX2-NEXT: pushq %r141787; AVX2-NEXT: pushq %r131788; AVX2-NEXT: pushq %r121789; AVX2-NEXT: pushq %rbx1790; AVX2-NEXT: andq $-32, %rsp1791; AVX2-NEXT: subq $96, %rsp1792; AVX2-NEXT: movl %r9d, %r11d1793; AVX2-NEXT: movl %r8d, %r10d1794; AVX2-NEXT: movl %ecx, %r9d1795; AVX2-NEXT: movl %edx, %r8d1796; AVX2-NEXT: # kill: def $esi killed $esi def $rsi1797; AVX2-NEXT: # kill: def $edi killed $edi def $rdi1798; AVX2-NEXT: movzbl 360(%rbp), %eax1799; AVX2-NEXT: movzbl 352(%rbp), %ecx1800; AVX2-NEXT: vmovd %ecx, %xmm41801; AVX2-NEXT: vpinsrb $1, %eax, %xmm4, %xmm41802; AVX2-NEXT: movzbl 368(%rbp), %eax1803; AVX2-NEXT: vpinsrb $2, %eax, %xmm4, %xmm41804; AVX2-NEXT: movzbl 376(%rbp), %eax1805; AVX2-NEXT: vpinsrb $3, %eax, %xmm4, %xmm41806; AVX2-NEXT: movzbl 384(%rbp), %eax1807; AVX2-NEXT: vpinsrb $4, %eax, %xmm4, %xmm41808; AVX2-NEXT: movzbl 392(%rbp), %eax1809; AVX2-NEXT: vpinsrb $5, %eax, %xmm4, %xmm41810; AVX2-NEXT: movzbl 400(%rbp), %eax1811; AVX2-NEXT: vpinsrb $6, %eax, %xmm4, %xmm41812; AVX2-NEXT: movzbl 408(%rbp), %eax1813; AVX2-NEXT: vpinsrb $7, %eax, %xmm4, %xmm41814; AVX2-NEXT: movzbl 416(%rbp), %eax1815; AVX2-NEXT: vpinsrb $8, %eax, %xmm4, %xmm41816; AVX2-NEXT: movzbl 424(%rbp), %eax1817; AVX2-NEXT: vpinsrb $9, %eax, %xmm4, %xmm41818; AVX2-NEXT: movzbl 432(%rbp), %eax1819; AVX2-NEXT: vpinsrb $10, %eax, %xmm4, %xmm41820; AVX2-NEXT: movzbl 440(%rbp), %eax1821; AVX2-NEXT: vpinsrb $11, %eax, %xmm4, %xmm41822; AVX2-NEXT: movzbl 448(%rbp), %eax1823; AVX2-NEXT: vpinsrb $12, %eax, %xmm4, %xmm41824; AVX2-NEXT: movzbl 456(%rbp), %eax1825; AVX2-NEXT: vpinsrb $13, %eax, %xmm4, %xmm41826; AVX2-NEXT: movzbl 464(%rbp), %eax1827; AVX2-NEXT: vpinsrb $14, %eax, %xmm4, %xmm41828; AVX2-NEXT: movzbl 472(%rbp), %eax1829; AVX2-NEXT: vpinsrb $15, %eax, %xmm4, %xmm41830; AVX2-NEXT: movzbl 224(%rbp), %eax1831; AVX2-NEXT: vmovd %eax, %xmm51832; AVX2-NEXT: movzbl 232(%rbp), %eax1833; AVX2-NEXT: vpinsrb $1, %eax, %xmm5, %xmm51834; AVX2-NEXT: movzbl 240(%rbp), %eax1835; AVX2-NEXT: vpinsrb $2, %eax, %xmm5, %xmm51836; AVX2-NEXT: movzbl 248(%rbp), %eax1837; AVX2-NEXT: vpinsrb $3, %eax, %xmm5, %xmm51838; AVX2-NEXT: movzbl 256(%rbp), %eax1839; AVX2-NEXT: vpinsrb $4, %eax, %xmm5, %xmm51840; AVX2-NEXT: movzbl 264(%rbp), %eax1841; AVX2-NEXT: vpinsrb $5, %eax, %xmm5, %xmm51842; AVX2-NEXT: movzbl 272(%rbp), %eax1843; AVX2-NEXT: vpinsrb $6, %eax, %xmm5, %xmm51844; AVX2-NEXT: movzbl 280(%rbp), %eax1845; AVX2-NEXT: vpinsrb $7, %eax, %xmm5, %xmm51846; AVX2-NEXT: movzbl 288(%rbp), %eax1847; AVX2-NEXT: vpinsrb $8, %eax, %xmm5, %xmm51848; AVX2-NEXT: movzbl 296(%rbp), %eax1849; AVX2-NEXT: vpinsrb $9, %eax, %xmm5, %xmm51850; AVX2-NEXT: movzbl 304(%rbp), %eax1851; AVX2-NEXT: vpinsrb $10, %eax, %xmm5, %xmm51852; AVX2-NEXT: movzbl 312(%rbp), %eax1853; AVX2-NEXT: vpinsrb $11, %eax, %xmm5, %xmm51854; AVX2-NEXT: movzbl 320(%rbp), %eax1855; AVX2-NEXT: vpinsrb $12, %eax, %xmm5, %xmm51856; AVX2-NEXT: movzbl 328(%rbp), %eax1857; AVX2-NEXT: vpinsrb $13, %eax, %xmm5, %xmm51858; AVX2-NEXT: movzbl 336(%rbp), %eax1859; AVX2-NEXT: vpinsrb $14, %eax, %xmm5, %xmm51860; AVX2-NEXT: movzbl 344(%rbp), %eax1861; AVX2-NEXT: vpinsrb $15, %eax, %xmm5, %xmm51862; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm5, %ymm41863; AVX2-NEXT: vmovd %edi, %xmm51864; AVX2-NEXT: vpinsrb $1, %esi, %xmm5, %xmm51865; AVX2-NEXT: vpinsrb $2, %edx, %xmm5, %xmm51866; AVX2-NEXT: vpinsrb $3, %r9d, %xmm5, %xmm51867; AVX2-NEXT: vpinsrb $4, %r10d, %xmm5, %xmm51868; AVX2-NEXT: vpinsrb $5, %r11d, %xmm5, %xmm51869; AVX2-NEXT: movzbl 16(%rbp), %ebx1870; AVX2-NEXT: vpinsrb $6, %ebx, %xmm5, %xmm51871; AVX2-NEXT: movzbl 24(%rbp), %r14d1872; AVX2-NEXT: vpinsrb $7, %r14d, %xmm5, %xmm51873; AVX2-NEXT: movzbl 32(%rbp), %r15d1874; AVX2-NEXT: vpinsrb $8, %r15d, %xmm5, %xmm51875; AVX2-NEXT: movzbl 40(%rbp), %r12d1876; AVX2-NEXT: vpinsrb $9, %r12d, %xmm5, %xmm51877; AVX2-NEXT: movzbl 48(%rbp), %r13d1878; AVX2-NEXT: vpinsrb $10, %r13d, %xmm5, %xmm51879; AVX2-NEXT: movzbl 56(%rbp), %eax1880; AVX2-NEXT: vpinsrb $11, %eax, %xmm5, %xmm51881; AVX2-NEXT: movzbl 64(%rbp), %eax1882; AVX2-NEXT: vpinsrb $12, %eax, %xmm5, %xmm51883; AVX2-NEXT: movzbl 72(%rbp), %eax1884; AVX2-NEXT: vpinsrb $13, %eax, %xmm5, %xmm51885; AVX2-NEXT: movzbl 80(%rbp), %eax1886; AVX2-NEXT: vpinsrb $14, %eax, %xmm5, %xmm51887; AVX2-NEXT: movzbl 88(%rbp), %eax1888; AVX2-NEXT: vpinsrb $15, %eax, %xmm5, %xmm51889; AVX2-NEXT: movzbl 96(%rbp), %eax1890; AVX2-NEXT: vmovd %eax, %xmm61891; AVX2-NEXT: movzbl 104(%rbp), %eax1892; AVX2-NEXT: vpinsrb $1, %eax, %xmm6, %xmm61893; AVX2-NEXT: movzbl 112(%rbp), %eax1894; AVX2-NEXT: vpinsrb $2, %eax, %xmm6, %xmm61895; AVX2-NEXT: movzbl 120(%rbp), %eax1896; AVX2-NEXT: vpinsrb $3, %eax, %xmm6, %xmm61897; AVX2-NEXT: movzbl 128(%rbp), %eax1898; AVX2-NEXT: vpinsrb $4, %eax, %xmm6, %xmm61899; AVX2-NEXT: movzbl 136(%rbp), %eax1900; AVX2-NEXT: vpinsrb $5, %eax, %xmm6, %xmm61901; AVX2-NEXT: movzbl 144(%rbp), %eax1902; AVX2-NEXT: vpinsrb $6, %eax, %xmm6, %xmm61903; AVX2-NEXT: movzbl 152(%rbp), %eax1904; AVX2-NEXT: vpinsrb $7, %eax, %xmm6, %xmm61905; AVX2-NEXT: movzbl 160(%rbp), %eax1906; AVX2-NEXT: vpinsrb $8, %eax, %xmm6, %xmm61907; AVX2-NEXT: movzbl 168(%rbp), %eax1908; AVX2-NEXT: vpinsrb $9, %eax, %xmm6, %xmm61909; AVX2-NEXT: movzbl 176(%rbp), %eax1910; AVX2-NEXT: vpinsrb $10, %eax, %xmm6, %xmm61911; AVX2-NEXT: movzbl 184(%rbp), %eax1912; AVX2-NEXT: vpinsrb $11, %eax, %xmm6, %xmm61913; AVX2-NEXT: movzbl 192(%rbp), %eax1914; AVX2-NEXT: vpinsrb $12, %eax, %xmm6, %xmm61915; AVX2-NEXT: movzbl 200(%rbp), %eax1916; AVX2-NEXT: vpinsrb $13, %eax, %xmm6, %xmm61917; AVX2-NEXT: movzbl 208(%rbp), %eax1918; AVX2-NEXT: vpinsrb $14, %eax, %xmm6, %xmm61919; AVX2-NEXT: movzbl 216(%rbp), %eax1920; AVX2-NEXT: vpinsrb $15, %eax, %xmm6, %xmm61921; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm5, %ymm51922; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]1923; AVX2-NEXT: vpand %ymm6, %ymm5, %ymm51924; AVX2-NEXT: vpand %ymm6, %ymm4, %ymm41925; AVX2-NEXT: vpaddb %ymm4, %ymm5, %ymm41926; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm51927; AVX2-NEXT: vpaddb %xmm5, %xmm4, %xmm41928; AVX2-NEXT: vpextrb $1, %xmm4, %eax1929; AVX2-NEXT: vmovd %xmm4, %ecx1930; AVX2-NEXT: addb %al, %cl1931; AVX2-NEXT: vpextrb $2, %xmm4, %edx1932; AVX2-NEXT: vpextrb $3, %xmm4, %eax1933; AVX2-NEXT: addb %dl, %al1934; AVX2-NEXT: addb %cl, %al1935; AVX2-NEXT: vpextrb $4, %xmm4, %ecx1936; AVX2-NEXT: vpextrb $5, %xmm4, %edx1937; AVX2-NEXT: addb %cl, %dl1938; AVX2-NEXT: vpextrb $6, %xmm4, %ecx1939; AVX2-NEXT: addb %dl, %cl1940; AVX2-NEXT: addb %al, %cl1941; AVX2-NEXT: vpextrb $7, %xmm4, %eax1942; AVX2-NEXT: vpextrb $8, %xmm4, %edx1943; AVX2-NEXT: addb %al, %dl1944; AVX2-NEXT: vpextrb $9, %xmm4, %eax1945; AVX2-NEXT: addb %dl, %al1946; AVX2-NEXT: vpextrb $10, %xmm4, %edx1947; AVX2-NEXT: addb %al, %dl1948; AVX2-NEXT: addb %cl, %dl1949; AVX2-NEXT: vpextrb $11, %xmm4, %eax1950; AVX2-NEXT: vpextrb $12, %xmm4, %ecx1951; AVX2-NEXT: addb %al, %cl1952; AVX2-NEXT: vpextrb $13, %xmm4, %eax1953; AVX2-NEXT: addb %cl, %al1954; AVX2-NEXT: vpextrb $14, %xmm4, %ecx1955; AVX2-NEXT: addb %al, %cl1956; AVX2-NEXT: vpextrb $15, %xmm4, %eax1957; AVX2-NEXT: addb %cl, %al1958; AVX2-NEXT: addb %dl, %al1959; AVX2-NEXT: vmovaps %ymm3, {{[0-9]+}}(%rsp)1960; AVX2-NEXT: vmovaps %ymm2, (%rsp)1961; AVX2-NEXT: movzbl %al, %eax1962; AVX2-NEXT: andl $63, %eax1963; AVX2-NEXT: movzbl (%rsp,%rax), %edx1964; AVX2-NEXT: vpextrb $0, %xmm0, (%rsp)1965; AVX2-NEXT: andl $1, %edi1966; AVX2-NEXT: vpextrb $1, %xmm0, (%rsp,%rdi)1967; AVX2-NEXT: andl $1, %esi1968; AVX2-NEXT: addq %rdi, %rsi1969; AVX2-NEXT: vpextrb $2, %xmm0, (%rsp,%rsi)1970; AVX2-NEXT: andl $1, %r8d1971; AVX2-NEXT: addq %rsi, %r81972; AVX2-NEXT: vpextrb $3, %xmm0, (%rsp,%r8)1973; AVX2-NEXT: andl $1, %r9d1974; AVX2-NEXT: addq %r8, %r91975; AVX2-NEXT: vpextrb $4, %xmm0, (%rsp,%r9)1976; AVX2-NEXT: andl $1, %r10d1977; AVX2-NEXT: addq %r9, %r101978; AVX2-NEXT: movl %r10d, %eax1979; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rax)1980; AVX2-NEXT: andl $1, %r11d1981; AVX2-NEXT: addq %r10, %r111982; AVX2-NEXT: movzbl %bl, %eax1983; AVX2-NEXT: andl $1, %eax1984; AVX2-NEXT: addq %r11, %rax1985; AVX2-NEXT: # kill: def $r11d killed $r11d killed $r11 def $r111986; AVX2-NEXT: andl $63, %r11d1987; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%r11)1988; AVX2-NEXT: movzbl %r14b, %ecx1989; AVX2-NEXT: andl $1, %ecx1990; AVX2-NEXT: addq %rax, %rcx1991; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax1992; AVX2-NEXT: andl $63, %eax1993; AVX2-NEXT: vpextrb $7, %xmm0, (%rsp,%rax)1994; AVX2-NEXT: movzbl %r15b, %eax1995; AVX2-NEXT: andl $1, %eax1996; AVX2-NEXT: addq %rcx, %rax1997; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx1998; AVX2-NEXT: andl $63, %ecx1999; AVX2-NEXT: vpextrb $8, %xmm0, (%rsp,%rcx)2000; AVX2-NEXT: movzbl %r12b, %ecx2001; AVX2-NEXT: andl $1, %ecx2002; AVX2-NEXT: addq %rax, %rcx2003; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2004; AVX2-NEXT: andl $63, %eax2005; AVX2-NEXT: vpextrb $9, %xmm0, (%rsp,%rax)2006; AVX2-NEXT: movzbl %r13b, %eax2007; AVX2-NEXT: andl $1, %eax2008; AVX2-NEXT: addq %rcx, %rax2009; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2010; AVX2-NEXT: andl $63, %ecx2011; AVX2-NEXT: vpextrb $10, %xmm0, (%rsp,%rcx)2012; AVX2-NEXT: movzbl 56(%rbp), %ecx2013; AVX2-NEXT: movzbl %cl, %ecx2014; AVX2-NEXT: andl $1, %ecx2015; AVX2-NEXT: addq %rax, %rcx2016; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2017; AVX2-NEXT: andl $63, %eax2018; AVX2-NEXT: vpextrb $11, %xmm0, (%rsp,%rax)2019; AVX2-NEXT: movzbl 64(%rbp), %eax2020; AVX2-NEXT: movzbl %al, %eax2021; AVX2-NEXT: andl $1, %eax2022; AVX2-NEXT: addq %rcx, %rax2023; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2024; AVX2-NEXT: andl $63, %ecx2025; AVX2-NEXT: vpextrb $12, %xmm0, (%rsp,%rcx)2026; AVX2-NEXT: movzbl 72(%rbp), %ecx2027; AVX2-NEXT: movzbl %cl, %ecx2028; AVX2-NEXT: andl $1, %ecx2029; AVX2-NEXT: addq %rax, %rcx2030; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2031; AVX2-NEXT: andl $63, %eax2032; AVX2-NEXT: vpextrb $13, %xmm0, (%rsp,%rax)2033; AVX2-NEXT: movzbl 80(%rbp), %eax2034; AVX2-NEXT: movzbl %al, %eax2035; AVX2-NEXT: andl $1, %eax2036; AVX2-NEXT: addq %rcx, %rax2037; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2038; AVX2-NEXT: andl $63, %ecx2039; AVX2-NEXT: vpextrb $14, %xmm0, (%rsp,%rcx)2040; AVX2-NEXT: movzbl 88(%rbp), %ecx2041; AVX2-NEXT: movzbl %cl, %ecx2042; AVX2-NEXT: andl $1, %ecx2043; AVX2-NEXT: addq %rax, %rcx2044; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2045; AVX2-NEXT: andl $63, %eax2046; AVX2-NEXT: vpextrb $15, %xmm0, (%rsp,%rax)2047; AVX2-NEXT: movzbl 96(%rbp), %eax2048; AVX2-NEXT: movzbl %al, %eax2049; AVX2-NEXT: andl $1, %eax2050; AVX2-NEXT: addq %rcx, %rax2051; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2052; AVX2-NEXT: andl $63, %ecx2053; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm02054; AVX2-NEXT: vpextrb $0, %xmm0, (%rsp,%rcx)2055; AVX2-NEXT: movzbl 104(%rbp), %ecx2056; AVX2-NEXT: movzbl %cl, %ecx2057; AVX2-NEXT: andl $1, %ecx2058; AVX2-NEXT: addq %rax, %rcx2059; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2060; AVX2-NEXT: andl $63, %eax2061; AVX2-NEXT: vpextrb $1, %xmm0, (%rsp,%rax)2062; AVX2-NEXT: movzbl 112(%rbp), %eax2063; AVX2-NEXT: movzbl %al, %eax2064; AVX2-NEXT: andl $1, %eax2065; AVX2-NEXT: addq %rcx, %rax2066; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2067; AVX2-NEXT: andl $63, %ecx2068; AVX2-NEXT: vpextrb $2, %xmm0, (%rsp,%rcx)2069; AVX2-NEXT: movzbl 120(%rbp), %ecx2070; AVX2-NEXT: movzbl %cl, %ecx2071; AVX2-NEXT: andl $1, %ecx2072; AVX2-NEXT: addq %rax, %rcx2073; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2074; AVX2-NEXT: andl $63, %eax2075; AVX2-NEXT: vpextrb $3, %xmm0, (%rsp,%rax)2076; AVX2-NEXT: movzbl 128(%rbp), %eax2077; AVX2-NEXT: movzbl %al, %eax2078; AVX2-NEXT: andl $1, %eax2079; AVX2-NEXT: addq %rcx, %rax2080; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2081; AVX2-NEXT: andl $63, %ecx2082; AVX2-NEXT: vpextrb $4, %xmm0, (%rsp,%rcx)2083; AVX2-NEXT: movzbl 136(%rbp), %ecx2084; AVX2-NEXT: movzbl %cl, %ecx2085; AVX2-NEXT: andl $1, %ecx2086; AVX2-NEXT: addq %rax, %rcx2087; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2088; AVX2-NEXT: andl $63, %eax2089; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rax)2090; AVX2-NEXT: movzbl 144(%rbp), %eax2091; AVX2-NEXT: movzbl %al, %eax2092; AVX2-NEXT: andl $1, %eax2093; AVX2-NEXT: addq %rcx, %rax2094; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2095; AVX2-NEXT: andl $63, %ecx2096; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%rcx)2097; AVX2-NEXT: movzbl 152(%rbp), %ecx2098; AVX2-NEXT: movzbl %cl, %ecx2099; AVX2-NEXT: andl $1, %ecx2100; AVX2-NEXT: addq %rax, %rcx2101; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2102; AVX2-NEXT: andl $63, %eax2103; AVX2-NEXT: vpextrb $7, %xmm0, (%rsp,%rax)2104; AVX2-NEXT: movzbl 160(%rbp), %eax2105; AVX2-NEXT: movzbl %al, %eax2106; AVX2-NEXT: andl $1, %eax2107; AVX2-NEXT: addq %rcx, %rax2108; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2109; AVX2-NEXT: andl $63, %ecx2110; AVX2-NEXT: vpextrb $8, %xmm0, (%rsp,%rcx)2111; AVX2-NEXT: movzbl 168(%rbp), %ecx2112; AVX2-NEXT: movzbl %cl, %ecx2113; AVX2-NEXT: andl $1, %ecx2114; AVX2-NEXT: addq %rax, %rcx2115; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2116; AVX2-NEXT: andl $63, %eax2117; AVX2-NEXT: vpextrb $9, %xmm0, (%rsp,%rax)2118; AVX2-NEXT: movzbl 176(%rbp), %eax2119; AVX2-NEXT: movzbl %al, %eax2120; AVX2-NEXT: andl $1, %eax2121; AVX2-NEXT: addq %rcx, %rax2122; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2123; AVX2-NEXT: andl $63, %ecx2124; AVX2-NEXT: vpextrb $10, %xmm0, (%rsp,%rcx)2125; AVX2-NEXT: movzbl 184(%rbp), %ecx2126; AVX2-NEXT: movzbl %cl, %ecx2127; AVX2-NEXT: andl $1, %ecx2128; AVX2-NEXT: addq %rax, %rcx2129; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2130; AVX2-NEXT: andl $63, %eax2131; AVX2-NEXT: vpextrb $11, %xmm0, (%rsp,%rax)2132; AVX2-NEXT: movzbl 192(%rbp), %eax2133; AVX2-NEXT: movzbl %al, %eax2134; AVX2-NEXT: andl $1, %eax2135; AVX2-NEXT: addq %rcx, %rax2136; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2137; AVX2-NEXT: andl $63, %ecx2138; AVX2-NEXT: vpextrb $12, %xmm0, (%rsp,%rcx)2139; AVX2-NEXT: movzbl 200(%rbp), %ecx2140; AVX2-NEXT: movzbl %cl, %ecx2141; AVX2-NEXT: andl $1, %ecx2142; AVX2-NEXT: addq %rax, %rcx2143; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2144; AVX2-NEXT: andl $63, %eax2145; AVX2-NEXT: vpextrb $13, %xmm0, (%rsp,%rax)2146; AVX2-NEXT: movzbl 208(%rbp), %eax2147; AVX2-NEXT: movzbl %al, %eax2148; AVX2-NEXT: andl $1, %eax2149; AVX2-NEXT: addq %rcx, %rax2150; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2151; AVX2-NEXT: andl $63, %ecx2152; AVX2-NEXT: vpextrb $14, %xmm0, (%rsp,%rcx)2153; AVX2-NEXT: movzbl 216(%rbp), %ecx2154; AVX2-NEXT: movzbl %cl, %ecx2155; AVX2-NEXT: andl $1, %ecx2156; AVX2-NEXT: addq %rax, %rcx2157; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2158; AVX2-NEXT: andl $63, %eax2159; AVX2-NEXT: vpextrb $15, %xmm0, (%rsp,%rax)2160; AVX2-NEXT: movzbl 224(%rbp), %eax2161; AVX2-NEXT: movzbl %al, %eax2162; AVX2-NEXT: andl $1, %eax2163; AVX2-NEXT: addq %rcx, %rax2164; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2165; AVX2-NEXT: andl $63, %ecx2166; AVX2-NEXT: vpextrb $0, %xmm1, (%rsp,%rcx)2167; AVX2-NEXT: movzbl 232(%rbp), %ecx2168; AVX2-NEXT: movzbl %cl, %ecx2169; AVX2-NEXT: andl $1, %ecx2170; AVX2-NEXT: addq %rax, %rcx2171; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2172; AVX2-NEXT: andl $63, %eax2173; AVX2-NEXT: vpextrb $1, %xmm1, (%rsp,%rax)2174; AVX2-NEXT: movzbl 240(%rbp), %eax2175; AVX2-NEXT: movzbl %al, %eax2176; AVX2-NEXT: andl $1, %eax2177; AVX2-NEXT: addq %rcx, %rax2178; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2179; AVX2-NEXT: andl $63, %ecx2180; AVX2-NEXT: vpextrb $2, %xmm1, (%rsp,%rcx)2181; AVX2-NEXT: movzbl 248(%rbp), %ecx2182; AVX2-NEXT: movzbl %cl, %ecx2183; AVX2-NEXT: andl $1, %ecx2184; AVX2-NEXT: addq %rax, %rcx2185; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2186; AVX2-NEXT: andl $63, %eax2187; AVX2-NEXT: vpextrb $3, %xmm1, (%rsp,%rax)2188; AVX2-NEXT: movzbl 256(%rbp), %eax2189; AVX2-NEXT: movzbl %al, %eax2190; AVX2-NEXT: andl $1, %eax2191; AVX2-NEXT: addq %rcx, %rax2192; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2193; AVX2-NEXT: andl $63, %ecx2194; AVX2-NEXT: vpextrb $4, %xmm1, (%rsp,%rcx)2195; AVX2-NEXT: movzbl 264(%rbp), %ecx2196; AVX2-NEXT: movzbl %cl, %ecx2197; AVX2-NEXT: andl $1, %ecx2198; AVX2-NEXT: addq %rax, %rcx2199; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2200; AVX2-NEXT: andl $63, %eax2201; AVX2-NEXT: vpextrb $5, %xmm1, (%rsp,%rax)2202; AVX2-NEXT: movzbl 272(%rbp), %eax2203; AVX2-NEXT: movzbl %al, %eax2204; AVX2-NEXT: andl $1, %eax2205; AVX2-NEXT: addq %rcx, %rax2206; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2207; AVX2-NEXT: andl $63, %ecx2208; AVX2-NEXT: vpextrb $6, %xmm1, (%rsp,%rcx)2209; AVX2-NEXT: movzbl 280(%rbp), %ecx2210; AVX2-NEXT: movzbl %cl, %ecx2211; AVX2-NEXT: andl $1, %ecx2212; AVX2-NEXT: addq %rax, %rcx2213; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2214; AVX2-NEXT: andl $63, %eax2215; AVX2-NEXT: vpextrb $7, %xmm1, (%rsp,%rax)2216; AVX2-NEXT: movzbl 288(%rbp), %eax2217; AVX2-NEXT: movzbl %al, %eax2218; AVX2-NEXT: andl $1, %eax2219; AVX2-NEXT: addq %rcx, %rax2220; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2221; AVX2-NEXT: andl $63, %ecx2222; AVX2-NEXT: vpextrb $8, %xmm1, (%rsp,%rcx)2223; AVX2-NEXT: movzbl 296(%rbp), %ecx2224; AVX2-NEXT: movzbl %cl, %ecx2225; AVX2-NEXT: andl $1, %ecx2226; AVX2-NEXT: addq %rax, %rcx2227; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2228; AVX2-NEXT: andl $63, %eax2229; AVX2-NEXT: vpextrb $9, %xmm1, (%rsp,%rax)2230; AVX2-NEXT: movzbl 304(%rbp), %eax2231; AVX2-NEXT: movzbl %al, %eax2232; AVX2-NEXT: andl $1, %eax2233; AVX2-NEXT: addq %rcx, %rax2234; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2235; AVX2-NEXT: andl $63, %ecx2236; AVX2-NEXT: vpextrb $10, %xmm1, (%rsp,%rcx)2237; AVX2-NEXT: movzbl 312(%rbp), %ecx2238; AVX2-NEXT: movzbl %cl, %ecx2239; AVX2-NEXT: andl $1, %ecx2240; AVX2-NEXT: addq %rax, %rcx2241; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2242; AVX2-NEXT: andl $63, %eax2243; AVX2-NEXT: vpextrb $11, %xmm1, (%rsp,%rax)2244; AVX2-NEXT: movzbl 320(%rbp), %eax2245; AVX2-NEXT: movzbl %al, %eax2246; AVX2-NEXT: andl $1, %eax2247; AVX2-NEXT: addq %rcx, %rax2248; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2249; AVX2-NEXT: andl $63, %ecx2250; AVX2-NEXT: vpextrb $12, %xmm1, (%rsp,%rcx)2251; AVX2-NEXT: movzbl 328(%rbp), %ecx2252; AVX2-NEXT: movzbl %cl, %ecx2253; AVX2-NEXT: andl $1, %ecx2254; AVX2-NEXT: addq %rax, %rcx2255; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2256; AVX2-NEXT: andl $63, %eax2257; AVX2-NEXT: vpextrb $13, %xmm1, (%rsp,%rax)2258; AVX2-NEXT: movzbl 336(%rbp), %eax2259; AVX2-NEXT: movzbl %al, %eax2260; AVX2-NEXT: andl $1, %eax2261; AVX2-NEXT: addq %rcx, %rax2262; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2263; AVX2-NEXT: andl $63, %ecx2264; AVX2-NEXT: vpextrb $14, %xmm1, (%rsp,%rcx)2265; AVX2-NEXT: movzbl 344(%rbp), %ecx2266; AVX2-NEXT: movzbl %cl, %ecx2267; AVX2-NEXT: andl $1, %ecx2268; AVX2-NEXT: addq %rax, %rcx2269; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2270; AVX2-NEXT: andl $63, %eax2271; AVX2-NEXT: vpextrb $15, %xmm1, (%rsp,%rax)2272; AVX2-NEXT: movzbl 352(%rbp), %eax2273; AVX2-NEXT: movzbl %al, %eax2274; AVX2-NEXT: andl $1, %eax2275; AVX2-NEXT: addq %rcx, %rax2276; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2277; AVX2-NEXT: andl $63, %ecx2278; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm02279; AVX2-NEXT: vpextrb $0, %xmm0, (%rsp,%rcx)2280; AVX2-NEXT: movzbl 360(%rbp), %ecx2281; AVX2-NEXT: movzbl %cl, %ecx2282; AVX2-NEXT: andl $1, %ecx2283; AVX2-NEXT: addq %rax, %rcx2284; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2285; AVX2-NEXT: andl $63, %eax2286; AVX2-NEXT: vpextrb $1, %xmm0, (%rsp,%rax)2287; AVX2-NEXT: movzbl 368(%rbp), %eax2288; AVX2-NEXT: movzbl %al, %eax2289; AVX2-NEXT: andl $1, %eax2290; AVX2-NEXT: addq %rcx, %rax2291; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2292; AVX2-NEXT: andl $63, %ecx2293; AVX2-NEXT: vpextrb $2, %xmm0, (%rsp,%rcx)2294; AVX2-NEXT: movzbl 376(%rbp), %ecx2295; AVX2-NEXT: movzbl %cl, %ecx2296; AVX2-NEXT: andl $1, %ecx2297; AVX2-NEXT: addq %rax, %rcx2298; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2299; AVX2-NEXT: andl $63, %eax2300; AVX2-NEXT: vpextrb $3, %xmm0, (%rsp,%rax)2301; AVX2-NEXT: movzbl 384(%rbp), %eax2302; AVX2-NEXT: movzbl %al, %eax2303; AVX2-NEXT: andl $1, %eax2304; AVX2-NEXT: addq %rcx, %rax2305; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2306; AVX2-NEXT: andl $63, %ecx2307; AVX2-NEXT: vpextrb $4, %xmm0, (%rsp,%rcx)2308; AVX2-NEXT: movzbl 392(%rbp), %ecx2309; AVX2-NEXT: movzbl %cl, %ecx2310; AVX2-NEXT: andl $1, %ecx2311; AVX2-NEXT: addq %rax, %rcx2312; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2313; AVX2-NEXT: andl $63, %eax2314; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rax)2315; AVX2-NEXT: movzbl 400(%rbp), %eax2316; AVX2-NEXT: movzbl %al, %eax2317; AVX2-NEXT: andl $1, %eax2318; AVX2-NEXT: addq %rcx, %rax2319; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2320; AVX2-NEXT: andl $63, %ecx2321; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%rcx)2322; AVX2-NEXT: movzbl 408(%rbp), %ecx2323; AVX2-NEXT: movzbl %cl, %ecx2324; AVX2-NEXT: andl $1, %ecx2325; AVX2-NEXT: addq %rax, %rcx2326; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2327; AVX2-NEXT: andl $63, %eax2328; AVX2-NEXT: vpextrb $7, %xmm0, (%rsp,%rax)2329; AVX2-NEXT: movzbl 416(%rbp), %eax2330; AVX2-NEXT: movzbl %al, %eax2331; AVX2-NEXT: andl $1, %eax2332; AVX2-NEXT: addq %rcx, %rax2333; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2334; AVX2-NEXT: andl $63, %ecx2335; AVX2-NEXT: vpextrb $8, %xmm0, (%rsp,%rcx)2336; AVX2-NEXT: movzbl 424(%rbp), %ecx2337; AVX2-NEXT: movzbl %cl, %ecx2338; AVX2-NEXT: andl $1, %ecx2339; AVX2-NEXT: addq %rax, %rcx2340; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2341; AVX2-NEXT: andl $63, %eax2342; AVX2-NEXT: vpextrb $9, %xmm0, (%rsp,%rax)2343; AVX2-NEXT: movzbl 432(%rbp), %eax2344; AVX2-NEXT: movzbl %al, %eax2345; AVX2-NEXT: andl $1, %eax2346; AVX2-NEXT: addq %rcx, %rax2347; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2348; AVX2-NEXT: andl $63, %ecx2349; AVX2-NEXT: vpextrb $10, %xmm0, (%rsp,%rcx)2350; AVX2-NEXT: movzbl 440(%rbp), %ecx2351; AVX2-NEXT: movzbl %cl, %ecx2352; AVX2-NEXT: andl $1, %ecx2353; AVX2-NEXT: addq %rax, %rcx2354; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2355; AVX2-NEXT: andl $63, %eax2356; AVX2-NEXT: vpextrb $11, %xmm0, (%rsp,%rax)2357; AVX2-NEXT: movzbl 448(%rbp), %eax2358; AVX2-NEXT: movzbl %al, %eax2359; AVX2-NEXT: andl $1, %eax2360; AVX2-NEXT: addq %rcx, %rax2361; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2362; AVX2-NEXT: andl $63, %ecx2363; AVX2-NEXT: vpextrb $12, %xmm0, (%rsp,%rcx)2364; AVX2-NEXT: movzbl 456(%rbp), %ecx2365; AVX2-NEXT: movzbl %cl, %ecx2366; AVX2-NEXT: andl $1, %ecx2367; AVX2-NEXT: addq %rax, %rcx2368; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2369; AVX2-NEXT: andl $63, %eax2370; AVX2-NEXT: vpextrb $13, %xmm0, (%rsp,%rax)2371; AVX2-NEXT: movzbl 464(%rbp), %eax2372; AVX2-NEXT: movzbl %al, %eax2373; AVX2-NEXT: andl $1, %eax2374; AVX2-NEXT: addq %rcx, %rax2375; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx2376; AVX2-NEXT: andl $63, %ecx2377; AVX2-NEXT: vpextrb $14, %xmm0, (%rsp,%rcx)2378; AVX2-NEXT: movzbl 472(%rbp), %ecx2379; AVX2-NEXT: movzbl %cl, %ecx2380; AVX2-NEXT: andl $1, %ecx2381; AVX2-NEXT: addq %rax, %rcx2382; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax2383; AVX2-NEXT: andl $63, %eax2384; AVX2-NEXT: vpextrb $15, %xmm0, (%rsp,%rax)2385; AVX2-NEXT: vpextrb $15, %xmm0, %eax2386; AVX2-NEXT: cmpq $64, %rcx2387; AVX2-NEXT: cmovbl %edx, %eax2388; AVX2-NEXT: cmpq $63, %rcx2389; AVX2-NEXT: movl $63, %edx2390; AVX2-NEXT: cmovbq %rcx, %rdx2391; AVX2-NEXT: movb %al, (%rsp,%rdx)2392; AVX2-NEXT: vmovaps (%rsp), %ymm02393; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm12394; AVX2-NEXT: leaq -40(%rbp), %rsp2395; AVX2-NEXT: popq %rbx2396; AVX2-NEXT: popq %r122397; AVX2-NEXT: popq %r132398; AVX2-NEXT: popq %r142399; AVX2-NEXT: popq %r152400; AVX2-NEXT: popq %rbp2401; AVX2-NEXT: retq2402;2403; AVX512F-LABEL: test_compress_v64i8:2404; AVX512F: # %bb.0:2405; AVX512F-NEXT: pushq %rbp2406; AVX512F-NEXT: movq %rsp, %rbp2407; AVX512F-NEXT: andq $-64, %rsp2408; AVX512F-NEXT: subq $256, %rsp # imm = 0x1002409; AVX512F-NEXT: movzbl 352(%rbp), %eax2410; AVX512F-NEXT: andl $1, %eax2411; AVX512F-NEXT: kmovw %eax, %k02412; AVX512F-NEXT: movzbl 360(%rbp), %eax2413; AVX512F-NEXT: kmovw %eax, %k12414; AVX512F-NEXT: kshiftlw $15, %k1, %k12415; AVX512F-NEXT: kshiftrw $14, %k1, %k12416; AVX512F-NEXT: korw %k1, %k0, %k02417; AVX512F-NEXT: movw $-5, %ax2418; AVX512F-NEXT: kmovw %eax, %k12419; AVX512F-NEXT: kandw %k1, %k0, %k02420; AVX512F-NEXT: kmovw %k1, %k32421; AVX512F-NEXT: movzbl 368(%rbp), %eax2422; AVX512F-NEXT: kmovw %eax, %k12423; AVX512F-NEXT: kshiftlw $15, %k1, %k12424; AVX512F-NEXT: kshiftrw $13, %k1, %k12425; AVX512F-NEXT: korw %k1, %k0, %k02426; AVX512F-NEXT: movw $-9, %ax2427; AVX512F-NEXT: kmovw %eax, %k52428; AVX512F-NEXT: kandw %k5, %k0, %k02429; AVX512F-NEXT: movzbl 376(%rbp), %eax2430; AVX512F-NEXT: kmovw %eax, %k12431; AVX512F-NEXT: kshiftlw $15, %k1, %k12432; AVX512F-NEXT: kshiftrw $12, %k1, %k12433; AVX512F-NEXT: korw %k1, %k0, %k02434; AVX512F-NEXT: movw $-17, %ax2435; AVX512F-NEXT: kmovw %eax, %k62436; AVX512F-NEXT: kandw %k6, %k0, %k02437; AVX512F-NEXT: movzbl 384(%rbp), %eax2438; AVX512F-NEXT: kmovw %eax, %k12439; AVX512F-NEXT: kshiftlw $15, %k1, %k12440; AVX512F-NEXT: kshiftrw $11, %k1, %k12441; AVX512F-NEXT: korw %k1, %k0, %k02442; AVX512F-NEXT: movw $-33, %ax2443; AVX512F-NEXT: kmovw %eax, %k72444; AVX512F-NEXT: kandw %k7, %k0, %k02445; AVX512F-NEXT: movzbl 392(%rbp), %eax2446; AVX512F-NEXT: kmovw %eax, %k12447; AVX512F-NEXT: kshiftlw $15, %k1, %k12448; AVX512F-NEXT: kshiftrw $10, %k1, %k12449; AVX512F-NEXT: korw %k1, %k0, %k02450; AVX512F-NEXT: movw $-65, %ax2451; AVX512F-NEXT: kmovw %eax, %k12452; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2453; AVX512F-NEXT: kandw %k1, %k0, %k02454; AVX512F-NEXT: movzbl 400(%rbp), %eax2455; AVX512F-NEXT: kmovw %eax, %k12456; AVX512F-NEXT: kshiftlw $15, %k1, %k12457; AVX512F-NEXT: kshiftrw $9, %k1, %k12458; AVX512F-NEXT: korw %k1, %k0, %k02459; AVX512F-NEXT: movw $-129, %ax2460; AVX512F-NEXT: kmovw %eax, %k12461; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2462; AVX512F-NEXT: kandw %k1, %k0, %k02463; AVX512F-NEXT: movzbl 408(%rbp), %eax2464; AVX512F-NEXT: kmovw %eax, %k12465; AVX512F-NEXT: kshiftlw $15, %k1, %k12466; AVX512F-NEXT: kshiftrw $8, %k1, %k12467; AVX512F-NEXT: korw %k1, %k0, %k02468; AVX512F-NEXT: movw $-257, %ax # imm = 0xFEFF2469; AVX512F-NEXT: kmovw %eax, %k12470; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2471; AVX512F-NEXT: kandw %k1, %k0, %k02472; AVX512F-NEXT: movzbl 416(%rbp), %eax2473; AVX512F-NEXT: kmovw %eax, %k12474; AVX512F-NEXT: kshiftlw $15, %k1, %k12475; AVX512F-NEXT: kshiftrw $7, %k1, %k12476; AVX512F-NEXT: korw %k1, %k0, %k02477; AVX512F-NEXT: movw $-513, %ax # imm = 0xFDFF2478; AVX512F-NEXT: kmovw %eax, %k12479; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2480; AVX512F-NEXT: kandw %k1, %k0, %k02481; AVX512F-NEXT: movzbl 424(%rbp), %eax2482; AVX512F-NEXT: kmovw %eax, %k12483; AVX512F-NEXT: kshiftlw $15, %k1, %k12484; AVX512F-NEXT: kshiftrw $6, %k1, %k12485; AVX512F-NEXT: korw %k1, %k0, %k02486; AVX512F-NEXT: movw $-1025, %ax # imm = 0xFBFF2487; AVX512F-NEXT: kmovw %eax, %k12488; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2489; AVX512F-NEXT: kandw %k1, %k0, %k02490; AVX512F-NEXT: movzbl 432(%rbp), %eax2491; AVX512F-NEXT: kmovw %eax, %k12492; AVX512F-NEXT: kshiftlw $15, %k1, %k12493; AVX512F-NEXT: kshiftrw $5, %k1, %k12494; AVX512F-NEXT: korw %k1, %k0, %k02495; AVX512F-NEXT: movw $-2049, %ax # imm = 0xF7FF2496; AVX512F-NEXT: kmovw %eax, %k12497; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2498; AVX512F-NEXT: kandw %k1, %k0, %k02499; AVX512F-NEXT: movzbl 440(%rbp), %eax2500; AVX512F-NEXT: kmovw %eax, %k12501; AVX512F-NEXT: kshiftlw $15, %k1, %k12502; AVX512F-NEXT: kshiftrw $4, %k1, %k12503; AVX512F-NEXT: korw %k1, %k0, %k02504; AVX512F-NEXT: movw $-4097, %ax # imm = 0xEFFF2505; AVX512F-NEXT: kmovw %eax, %k12506; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2507; AVX512F-NEXT: kandw %k1, %k0, %k02508; AVX512F-NEXT: movzbl 448(%rbp), %eax2509; AVX512F-NEXT: kmovw %eax, %k12510; AVX512F-NEXT: kshiftlw $15, %k1, %k12511; AVX512F-NEXT: kshiftrw $3, %k1, %k12512; AVX512F-NEXT: korw %k1, %k0, %k02513; AVX512F-NEXT: movw $-8193, %ax # imm = 0xDFFF2514; AVX512F-NEXT: kmovw %eax, %k12515; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2516; AVX512F-NEXT: kandw %k1, %k0, %k02517; AVX512F-NEXT: movzbl 456(%rbp), %eax2518; AVX512F-NEXT: kmovw %eax, %k12519; AVX512F-NEXT: kshiftlw $15, %k1, %k12520; AVX512F-NEXT: kshiftrw $2, %k1, %k12521; AVX512F-NEXT: korw %k1, %k0, %k12522; AVX512F-NEXT: movw $-16385, %ax # imm = 0xBFFF2523; AVX512F-NEXT: kmovw %eax, %k42524; AVX512F-NEXT: kandw %k4, %k1, %k12525; AVX512F-NEXT: kmovw %k4, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2526; AVX512F-NEXT: movzbl 464(%rbp), %eax2527; AVX512F-NEXT: kmovw %eax, %k22528; AVX512F-NEXT: kshiftlw $14, %k2, %k22529; AVX512F-NEXT: korw %k2, %k1, %k12530; AVX512F-NEXT: kshiftlw $1, %k1, %k12531; AVX512F-NEXT: kshiftrw $1, %k1, %k12532; AVX512F-NEXT: movzbl 472(%rbp), %eax2533; AVX512F-NEXT: kmovw %eax, %k22534; AVX512F-NEXT: kshiftlw $15, %k2, %k22535; AVX512F-NEXT: korw %k2, %k1, %k12536; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2537; AVX512F-NEXT: movzbl 224(%rbp), %eax2538; AVX512F-NEXT: andl $1, %eax2539; AVX512F-NEXT: movzbl 232(%rbp), %r10d2540; AVX512F-NEXT: kmovw %r10d, %k12541; AVX512F-NEXT: kshiftlw $15, %k1, %k12542; AVX512F-NEXT: kshiftrw $14, %k1, %k12543; AVX512F-NEXT: kmovw %eax, %k22544; AVX512F-NEXT: korw %k1, %k2, %k12545; AVX512F-NEXT: kmovw %k3, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2546; AVX512F-NEXT: kandw %k3, %k1, %k12547; AVX512F-NEXT: movzbl 240(%rbp), %eax2548; AVX512F-NEXT: kmovw %eax, %k22549; AVX512F-NEXT: kshiftlw $15, %k2, %k22550; AVX512F-NEXT: kshiftrw $13, %k2, %k22551; AVX512F-NEXT: korw %k2, %k1, %k12552; AVX512F-NEXT: kandw %k5, %k1, %k12553; AVX512F-NEXT: movzbl 248(%rbp), %eax2554; AVX512F-NEXT: kmovw %eax, %k22555; AVX512F-NEXT: kshiftlw $15, %k2, %k22556; AVX512F-NEXT: kshiftrw $12, %k2, %k22557; AVX512F-NEXT: korw %k2, %k1, %k12558; AVX512F-NEXT: kandw %k6, %k1, %k12559; AVX512F-NEXT: movzbl 256(%rbp), %eax2560; AVX512F-NEXT: kmovw %eax, %k22561; AVX512F-NEXT: kshiftlw $15, %k2, %k22562; AVX512F-NEXT: kshiftrw $11, %k2, %k22563; AVX512F-NEXT: korw %k2, %k1, %k12564; AVX512F-NEXT: kandw %k7, %k1, %k12565; AVX512F-NEXT: movzbl 264(%rbp), %eax2566; AVX512F-NEXT: kmovw %eax, %k22567; AVX512F-NEXT: kshiftlw $15, %k2, %k22568; AVX512F-NEXT: kshiftrw $10, %k2, %k22569; AVX512F-NEXT: korw %k2, %k1, %k12570; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload2571; AVX512F-NEXT: kandw %k0, %k1, %k12572; AVX512F-NEXT: movzbl 272(%rbp), %eax2573; AVX512F-NEXT: kmovw %eax, %k22574; AVX512F-NEXT: kshiftlw $15, %k2, %k22575; AVX512F-NEXT: kshiftrw $9, %k2, %k22576; AVX512F-NEXT: korw %k2, %k1, %k02577; AVX512F-NEXT: kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2578; AVX512F-NEXT: movzbl 280(%rbp), %eax2579; AVX512F-NEXT: kmovw %eax, %k12580; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2581; AVX512F-NEXT: kshiftlw $15, %k1, %k12582; AVX512F-NEXT: kshiftrw $8, %k1, %k12583; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload2584; AVX512F-NEXT: kandw %k2, %k0, %k22585; AVX512F-NEXT: korw %k1, %k2, %k12586; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload2587; AVX512F-NEXT: kandw %k0, %k1, %k12588; AVX512F-NEXT: movzbl 288(%rbp), %eax2589; AVX512F-NEXT: kmovw %eax, %k02590; AVX512F-NEXT: kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2591; AVX512F-NEXT: kshiftlw $15, %k0, %k22592; AVX512F-NEXT: kshiftrw $7, %k2, %k22593; AVX512F-NEXT: korw %k2, %k1, %k12594; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload2595; AVX512F-NEXT: kandw %k0, %k1, %k12596; AVX512F-NEXT: movzbl 296(%rbp), %eax2597; AVX512F-NEXT: kmovw %eax, %k22598; AVX512F-NEXT: kshiftlw $15, %k2, %k02599; AVX512F-NEXT: kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2600; AVX512F-NEXT: kshiftrw $6, %k0, %k22601; AVX512F-NEXT: korw %k2, %k1, %k12602; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload2603; AVX512F-NEXT: kandw %k0, %k1, %k12604; AVX512F-NEXT: movzbl 304(%rbp), %eax2605; AVX512F-NEXT: kmovw %eax, %k22606; AVX512F-NEXT: kshiftlw $15, %k2, %k02607; AVX512F-NEXT: kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2608; AVX512F-NEXT: kshiftrw $5, %k0, %k22609; AVX512F-NEXT: korw %k2, %k1, %k12610; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload2611; AVX512F-NEXT: kandw %k0, %k1, %k12612; AVX512F-NEXT: movzbl 312(%rbp), %eax2613; AVX512F-NEXT: kmovw %eax, %k22614; AVX512F-NEXT: kshiftlw $15, %k2, %k02615; AVX512F-NEXT: kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2616; AVX512F-NEXT: kshiftrw $4, %k0, %k22617; AVX512F-NEXT: korw %k2, %k1, %k12618; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload2619; AVX512F-NEXT: kandw %k0, %k1, %k12620; AVX512F-NEXT: movzbl 320(%rbp), %eax2621; AVX512F-NEXT: kmovw %eax, %k22622; AVX512F-NEXT: kshiftlw $15, %k2, %k02623; AVX512F-NEXT: kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2624; AVX512F-NEXT: kshiftrw $3, %k0, %k22625; AVX512F-NEXT: korw %k2, %k1, %k12626; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload2627; AVX512F-NEXT: kandw %k0, %k1, %k12628; AVX512F-NEXT: movzbl 328(%rbp), %eax2629; AVX512F-NEXT: kmovw %eax, %k22630; AVX512F-NEXT: kshiftlw $15, %k2, %k22631; AVX512F-NEXT: kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2632; AVX512F-NEXT: kshiftrw $2, %k2, %k22633; AVX512F-NEXT: korw %k2, %k1, %k12634; AVX512F-NEXT: kandw %k4, %k1, %k12635; AVX512F-NEXT: movzbl 336(%rbp), %eax2636; AVX512F-NEXT: kmovw %eax, %k22637; AVX512F-NEXT: kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2638; AVX512F-NEXT: kshiftlw $14, %k2, %k22639; AVX512F-NEXT: korw %k2, %k1, %k12640; AVX512F-NEXT: kshiftlw $1, %k1, %k12641; AVX512F-NEXT: kshiftrw $1, %k1, %k12642; AVX512F-NEXT: movzbl 344(%rbp), %eax2643; AVX512F-NEXT: kmovw %eax, %k22644; AVX512F-NEXT: kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2645; AVX512F-NEXT: kshiftlw $15, %k2, %k22646; AVX512F-NEXT: korw %k2, %k1, %k12647; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2648; AVX512F-NEXT: movzbl 96(%rbp), %eax2649; AVX512F-NEXT: andl $1, %eax2650; AVX512F-NEXT: movzbl 104(%rbp), %r10d2651; AVX512F-NEXT: kmovw %r10d, %k12652; AVX512F-NEXT: kshiftlw $15, %k1, %k12653; AVX512F-NEXT: kshiftrw $14, %k1, %k12654; AVX512F-NEXT: kmovw %eax, %k22655; AVX512F-NEXT: korw %k1, %k2, %k12656; AVX512F-NEXT: kandw %k3, %k1, %k12657; AVX512F-NEXT: movzbl 112(%rbp), %eax2658; AVX512F-NEXT: kmovw %eax, %k22659; AVX512F-NEXT: kshiftlw $15, %k2, %k22660; AVX512F-NEXT: kshiftrw $13, %k2, %k22661; AVX512F-NEXT: korw %k2, %k1, %k12662; AVX512F-NEXT: kmovw %k5, %k42663; AVX512F-NEXT: kmovw %k5, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2664; AVX512F-NEXT: kandw %k5, %k1, %k12665; AVX512F-NEXT: movzbl 120(%rbp), %eax2666; AVX512F-NEXT: kmovw %eax, %k22667; AVX512F-NEXT: kshiftlw $15, %k2, %k22668; AVX512F-NEXT: kshiftrw $12, %k2, %k22669; AVX512F-NEXT: korw %k2, %k1, %k12670; AVX512F-NEXT: kmovw %k6, %k52671; AVX512F-NEXT: kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2672; AVX512F-NEXT: kandw %k6, %k1, %k12673; AVX512F-NEXT: movzbl 128(%rbp), %eax2674; AVX512F-NEXT: kmovw %eax, %k22675; AVX512F-NEXT: kshiftlw $15, %k2, %k22676; AVX512F-NEXT: kshiftrw $11, %k2, %k22677; AVX512F-NEXT: korw %k2, %k1, %k12678; AVX512F-NEXT: kmovw %k7, %k62679; AVX512F-NEXT: kmovw %k7, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2680; AVX512F-NEXT: kandw %k7, %k1, %k12681; AVX512F-NEXT: movzbl 136(%rbp), %eax2682; AVX512F-NEXT: kmovw %eax, %k22683; AVX512F-NEXT: kshiftlw $15, %k2, %k22684; AVX512F-NEXT: kshiftrw $10, %k2, %k22685; AVX512F-NEXT: korw %k2, %k1, %k12686; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload2687; AVX512F-NEXT: kandw %k7, %k1, %k12688; AVX512F-NEXT: movzbl 144(%rbp), %eax2689; AVX512F-NEXT: kmovw %eax, %k22690; AVX512F-NEXT: kshiftlw $15, %k2, %k22691; AVX512F-NEXT: kshiftrw $9, %k2, %k22692; AVX512F-NEXT: korw %k2, %k1, %k12693; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload2694; AVX512F-NEXT: kandw %k2, %k1, %k12695; AVX512F-NEXT: movzbl 152(%rbp), %eax2696; AVX512F-NEXT: kmovw %eax, %k22697; AVX512F-NEXT: kshiftlw $15, %k2, %k22698; AVX512F-NEXT: kshiftrw $8, %k2, %k22699; AVX512F-NEXT: korw %k2, %k1, %k12700; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload2701; AVX512F-NEXT: kandw %k3, %k1, %k12702; AVX512F-NEXT: movzbl 160(%rbp), %eax2703; AVX512F-NEXT: kmovw %eax, %k22704; AVX512F-NEXT: kshiftlw $15, %k2, %k22705; AVX512F-NEXT: kshiftrw $7, %k2, %k22706; AVX512F-NEXT: korw %k2, %k1, %k12707; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload2708; AVX512F-NEXT: kandw %k2, %k1, %k12709; AVX512F-NEXT: movzbl 168(%rbp), %eax2710; AVX512F-NEXT: kmovw %eax, %k22711; AVX512F-NEXT: kshiftlw $15, %k2, %k22712; AVX512F-NEXT: kshiftrw $6, %k2, %k22713; AVX512F-NEXT: korw %k2, %k1, %k12714; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload2715; AVX512F-NEXT: kandw %k2, %k1, %k12716; AVX512F-NEXT: movzbl 176(%rbp), %eax2717; AVX512F-NEXT: kmovw %eax, %k22718; AVX512F-NEXT: kshiftlw $15, %k2, %k22719; AVX512F-NEXT: kshiftrw $5, %k2, %k22720; AVX512F-NEXT: korw %k2, %k1, %k12721; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload2722; AVX512F-NEXT: kandw %k2, %k1, %k12723; AVX512F-NEXT: movzbl 184(%rbp), %eax2724; AVX512F-NEXT: kmovw %eax, %k22725; AVX512F-NEXT: kshiftlw $15, %k2, %k22726; AVX512F-NEXT: kshiftrw $4, %k2, %k22727; AVX512F-NEXT: korw %k2, %k1, %k12728; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload2729; AVX512F-NEXT: kandw %k2, %k1, %k12730; AVX512F-NEXT: movzbl 192(%rbp), %eax2731; AVX512F-NEXT: kmovw %eax, %k22732; AVX512F-NEXT: kshiftlw $15, %k2, %k22733; AVX512F-NEXT: kshiftrw $3, %k2, %k22734; AVX512F-NEXT: korw %k2, %k1, %k12735; AVX512F-NEXT: kandw %k0, %k1, %k12736; AVX512F-NEXT: movzbl 200(%rbp), %eax2737; AVX512F-NEXT: kmovw %eax, %k22738; AVX512F-NEXT: kshiftlw $15, %k2, %k22739; AVX512F-NEXT: kshiftrw $2, %k2, %k22740; AVX512F-NEXT: korw %k2, %k1, %k12741; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload2742; AVX512F-NEXT: kandw %k2, %k1, %k12743; AVX512F-NEXT: movzbl 208(%rbp), %eax2744; AVX512F-NEXT: kmovw %eax, %k22745; AVX512F-NEXT: kshiftlw $14, %k2, %k22746; AVX512F-NEXT: korw %k2, %k1, %k12747; AVX512F-NEXT: kshiftlw $1, %k1, %k12748; AVX512F-NEXT: kshiftrw $1, %k1, %k12749; AVX512F-NEXT: movzbl 216(%rbp), %eax2750; AVX512F-NEXT: kmovw %eax, %k22751; AVX512F-NEXT: kshiftlw $15, %k2, %k22752; AVX512F-NEXT: korw %k2, %k1, %k12753; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2754; AVX512F-NEXT: andl $1, %edi2755; AVX512F-NEXT: kmovw %esi, %k12756; AVX512F-NEXT: kshiftlw $15, %k1, %k12757; AVX512F-NEXT: kshiftrw $14, %k1, %k12758; AVX512F-NEXT: kmovw %edi, %k22759; AVX512F-NEXT: korw %k1, %k2, %k12760; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload2761; AVX512F-NEXT: kandw %k2, %k1, %k12762; AVX512F-NEXT: kmovw %edx, %k22763; AVX512F-NEXT: kshiftlw $15, %k2, %k22764; AVX512F-NEXT: kshiftrw $13, %k2, %k22765; AVX512F-NEXT: korw %k2, %k1, %k12766; AVX512F-NEXT: kandw %k4, %k1, %k12767; AVX512F-NEXT: kmovw %ecx, %k22768; AVX512F-NEXT: kshiftlw $15, %k2, %k22769; AVX512F-NEXT: kshiftrw $12, %k2, %k22770; AVX512F-NEXT: korw %k2, %k1, %k12771; AVX512F-NEXT: kandw %k5, %k1, %k12772; AVX512F-NEXT: kmovw %r8d, %k22773; AVX512F-NEXT: kshiftlw $15, %k2, %k22774; AVX512F-NEXT: kshiftrw $11, %k2, %k22775; AVX512F-NEXT: korw %k2, %k1, %k12776; AVX512F-NEXT: kandw %k6, %k1, %k12777; AVX512F-NEXT: kmovw %r9d, %k22778; AVX512F-NEXT: kshiftlw $15, %k2, %k22779; AVX512F-NEXT: kshiftrw $10, %k2, %k22780; AVX512F-NEXT: korw %k2, %k1, %k12781; AVX512F-NEXT: kandw %k7, %k1, %k12782; AVX512F-NEXT: movzbl 16(%rbp), %eax2783; AVX512F-NEXT: kmovw %eax, %k22784; AVX512F-NEXT: kshiftlw $15, %k2, %k22785; AVX512F-NEXT: kshiftrw $9, %k2, %k22786; AVX512F-NEXT: korw %k2, %k1, %k22787; AVX512F-NEXT: kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2788; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload2789; AVX512F-NEXT: kandw %k1, %k2, %k12790; AVX512F-NEXT: movzbl 24(%rbp), %eax2791; AVX512F-NEXT: kmovw %eax, %k22792; AVX512F-NEXT: kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2793; AVX512F-NEXT: kshiftlw $15, %k2, %k22794; AVX512F-NEXT: kshiftrw $8, %k2, %k22795; AVX512F-NEXT: korw %k2, %k1, %k12796; AVX512F-NEXT: kandw %k3, %k1, %k12797; AVX512F-NEXT: movzbl 32(%rbp), %eax2798; AVX512F-NEXT: kmovw %eax, %k22799; AVX512F-NEXT: kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2800; AVX512F-NEXT: kshiftlw $15, %k2, %k22801; AVX512F-NEXT: kshiftrw $7, %k2, %k22802; AVX512F-NEXT: korw %k2, %k1, %k12803; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload2804; AVX512F-NEXT: kandw %k2, %k1, %k12805; AVX512F-NEXT: movzbl 40(%rbp), %eax2806; AVX512F-NEXT: kmovw %eax, %k22807; AVX512F-NEXT: kshiftlw $15, %k2, %k22808; AVX512F-NEXT: kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2809; AVX512F-NEXT: kshiftrw $6, %k2, %k22810; AVX512F-NEXT: korw %k2, %k1, %k12811; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload2812; AVX512F-NEXT: kandw %k2, %k1, %k12813; AVX512F-NEXT: movzbl 48(%rbp), %eax2814; AVX512F-NEXT: kmovw %eax, %k22815; AVX512F-NEXT: kshiftlw $15, %k2, %k52816; AVX512F-NEXT: kshiftrw $5, %k5, %k22817; AVX512F-NEXT: korw %k2, %k1, %k12818; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload2819; AVX512F-NEXT: kandw %k2, %k1, %k12820; AVX512F-NEXT: movzbl 56(%rbp), %eax2821; AVX512F-NEXT: kmovw %eax, %k22822; AVX512F-NEXT: kshiftlw $15, %k2, %k42823; AVX512F-NEXT: kshiftrw $4, %k4, %k22824; AVX512F-NEXT: korw %k2, %k1, %k12825; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload2826; AVX512F-NEXT: kandw %k2, %k1, %k12827; AVX512F-NEXT: movzbl 64(%rbp), %eax2828; AVX512F-NEXT: kmovw %eax, %k22829; AVX512F-NEXT: kshiftlw $15, %k2, %k32830; AVX512F-NEXT: kshiftrw $3, %k3, %k22831; AVX512F-NEXT: korw %k2, %k1, %k12832; AVX512F-NEXT: kandw %k0, %k1, %k12833; AVX512F-NEXT: movzbl 72(%rbp), %eax2834; AVX512F-NEXT: kmovw %eax, %k22835; AVX512F-NEXT: kshiftlw $15, %k2, %k22836; AVX512F-NEXT: kshiftrw $2, %k2, %k02837; AVX512F-NEXT: korw %k0, %k1, %k02838; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload2839; AVX512F-NEXT: kandw %k1, %k0, %k02840; AVX512F-NEXT: movzbl 80(%rbp), %eax2841; AVX512F-NEXT: kmovw %eax, %k12842; AVX512F-NEXT: kshiftlw $14, %k1, %k72843; AVX512F-NEXT: korw %k7, %k0, %k02844; AVX512F-NEXT: kshiftlw $1, %k0, %k02845; AVX512F-NEXT: kshiftrw $1, %k0, %k72846; AVX512F-NEXT: movzbl 88(%rbp), %eax2847; AVX512F-NEXT: kmovw %eax, %k02848; AVX512F-NEXT: kshiftlw $15, %k0, %k62849; AVX512F-NEXT: korw %k6, %k7, %k62850; AVX512F-NEXT: kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2851; AVX512F-NEXT: movw $-3, %ax2852; AVX512F-NEXT: kmovw %eax, %k62853; AVX512F-NEXT: kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill2854; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload2855; AVX512F-NEXT: kandw %k6, %k7, %k62856; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload2857; AVX512F-NEXT: kshiftrw $14, %k7, %k72858; AVX512F-NEXT: korw %k7, %k6, %k62859; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload2860; AVX512F-NEXT: kandw %k7, %k6, %k62861; AVX512F-NEXT: kshiftrw $13, %k5, %k52862; AVX512F-NEXT: korw %k5, %k6, %k52863; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 # 2-byte Reload2864; AVX512F-NEXT: kandw %k6, %k5, %k52865; AVX512F-NEXT: kshiftrw $12, %k4, %k42866; AVX512F-NEXT: korw %k4, %k5, %k42867; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload2868; AVX512F-NEXT: kandw %k5, %k4, %k42869; AVX512F-NEXT: kshiftrw $11, %k3, %k32870; AVX512F-NEXT: korw %k3, %k4, %k32871; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 # 2-byte Reload2872; AVX512F-NEXT: kandw %k4, %k3, %k32873; AVX512F-NEXT: kshiftrw $10, %k2, %k22874; AVX512F-NEXT: korw %k2, %k3, %k22875; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload2876; AVX512F-NEXT: kandw %k3, %k2, %k22877; AVX512F-NEXT: kshiftlw $6, %k1, %k12878; AVX512F-NEXT: korw %k1, %k2, %k12879; AVX512F-NEXT: kshiftlw $9, %k1, %k12880; AVX512F-NEXT: kshiftrw $9, %k1, %k12881; AVX512F-NEXT: kshiftlw $7, %k0, %k02882; AVX512F-NEXT: korw %k0, %k1, %k02883; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload2884; AVX512F-NEXT: kshiftlw $9, %k1, %k12885; AVX512F-NEXT: kshiftrw $9, %k1, %k12886; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload2887; AVX512F-NEXT: kshiftlw $7, %k2, %k22888; AVX512F-NEXT: korw %k2, %k1, %k12889; AVX512F-NEXT: kxorw %k0, %k1, %k02890; AVX512F-NEXT: kshiftrw $4, %k0, %k12891; AVX512F-NEXT: kxorw %k1, %k0, %k02892; AVX512F-NEXT: kshiftrw $2, %k0, %k12893; AVX512F-NEXT: kxorw %k1, %k0, %k02894; AVX512F-NEXT: kshiftrw $1, %k0, %k12895; AVX512F-NEXT: kxorw %k1, %k0, %k02896; AVX512F-NEXT: kmovw %k0, %eax2897; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload2898; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload2899; AVX512F-NEXT: kandw %k1, %k0, %k02900; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload2901; AVX512F-NEXT: kshiftrw $14, %k1, %k12902; AVX512F-NEXT: korw %k1, %k0, %k02903; AVX512F-NEXT: kandw %k7, %k0, %k02904; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload2905; AVX512F-NEXT: kshiftrw $13, %k1, %k12906; AVX512F-NEXT: korw %k1, %k0, %k02907; AVX512F-NEXT: kandw %k6, %k0, %k02908; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload2909; AVX512F-NEXT: kshiftrw $12, %k1, %k12910; AVX512F-NEXT: korw %k1, %k0, %k02911; AVX512F-NEXT: kandw %k5, %k0, %k02912; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload2913; AVX512F-NEXT: kshiftrw $11, %k1, %k12914; AVX512F-NEXT: korw %k1, %k0, %k02915; AVX512F-NEXT: kandw %k4, %k0, %k02916; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload2917; AVX512F-NEXT: kshiftrw $10, %k1, %k12918; AVX512F-NEXT: korw %k1, %k0, %k02919; AVX512F-NEXT: kandw %k3, %k0, %k02920; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload2921; AVX512F-NEXT: kshiftlw $6, %k1, %k12922; AVX512F-NEXT: korw %k1, %k0, %k02923; AVX512F-NEXT: kshiftlw $9, %k0, %k02924; AVX512F-NEXT: kshiftrw $9, %k0, %k02925; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload2926; AVX512F-NEXT: kshiftlw $7, %k1, %k12927; AVX512F-NEXT: korw %k1, %k0, %k02928; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload2929; AVX512F-NEXT: kshiftlw $9, %k1, %k12930; AVX512F-NEXT: kshiftrw $9, %k1, %k12931; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload2932; AVX512F-NEXT: kshiftlw $7, %k2, %k22933; AVX512F-NEXT: korw %k2, %k1, %k12934; AVX512F-NEXT: kxorw %k0, %k1, %k02935; AVX512F-NEXT: kshiftrw $4, %k0, %k12936; AVX512F-NEXT: kxorw %k1, %k0, %k02937; AVX512F-NEXT: kshiftrw $2, %k0, %k12938; AVX512F-NEXT: kxorw %k1, %k0, %k02939; AVX512F-NEXT: kshiftrw $1, %k0, %k12940; AVX512F-NEXT: kxorw %k1, %k0, %k02941; AVX512F-NEXT: kmovw %k0, %ecx2942; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload2943; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload2944; AVX512F-NEXT: kxorw %k2, %k3, %k02945; AVX512F-NEXT: kshiftrw $8, %k0, %k12946; AVX512F-NEXT: kxorw %k1, %k0, %k02947; AVX512F-NEXT: kshiftrw $4, %k0, %k12948; AVX512F-NEXT: kxorw %k1, %k0, %k02949; AVX512F-NEXT: kshiftrw $2, %k0, %k12950; AVX512F-NEXT: kxorw %k1, %k0, %k02951; AVX512F-NEXT: kshiftrw $1, %k0, %k12952; AVX512F-NEXT: kxorw %k1, %k0, %k02953; AVX512F-NEXT: kmovw %k0, %edx2954; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm22955; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero2956; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload2957; AVX512F-NEXT: vpcompressd %zmm3, %zmm3 {%k1} {z}2958; AVX512F-NEXT: vpternlogd {{.*#+}} zmm4 {%k1} {z} = -12959; AVX512F-NEXT: vextracti128 $1, %ymm2, %xmm22960; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero2961; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload2962; AVX512F-NEXT: vpcompressd %zmm2, %zmm2 {%k1} {z}2963; AVX512F-NEXT: vpternlogd {{.*#+}} zmm5 {%k1} {z} = -12964; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm6 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero2965; AVX512F-NEXT: vpcompressd %zmm6, %zmm6 {%k3} {z}2966; AVX512F-NEXT: vpternlogd {{.*#+}} zmm7 {%k3} {z} = -12967; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm02968; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero2969; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k2} {z}2970; AVX512F-NEXT: vpternlogd {{.*#+}} zmm8 {%k2} {z} = -12971; AVX512F-NEXT: vpmovdb %zmm6, {{[0-9]+}}(%rsp)2972; AVX512F-NEXT: andl $31, %eax2973; AVX512F-NEXT: vpmovdb %zmm0, 64(%rsp,%rax)2974; AVX512F-NEXT: vpmovdb %zmm3, {{[0-9]+}}(%rsp)2975; AVX512F-NEXT: andl $31, %ecx2976; AVX512F-NEXT: vpmovdb %zmm2, 96(%rsp,%rcx)2977; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm02978; AVX512F-NEXT: vmovaps %ymm0, {{[0-9]+}}(%rsp)2979; AVX512F-NEXT: andl $63, %edx2980; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm02981; AVX512F-NEXT: vmovaps %ymm0, 128(%rsp,%rdx)2982; AVX512F-NEXT: vpmovdb %zmm4, %xmm02983; AVX512F-NEXT: vpmovdb %zmm5, %xmm22984; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm02985; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm22986; AVX512F-NEXT: vpblendvb %ymm0, {{[0-9]+}}(%rsp), %ymm2, %ymm02987; AVX512F-NEXT: vpmovdb %zmm7, %xmm22988; AVX512F-NEXT: vpmovdb %zmm8, %xmm32989; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm22990; AVX512F-NEXT: vpblendvb %ymm2, {{[0-9]+}}(%rsp), %ymm1, %ymm12991; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm02992; AVX512F-NEXT: movq %rbp, %rsp2993; AVX512F-NEXT: popq %rbp2994; AVX512F-NEXT: retq2995;2996; AVX512VL-LABEL: test_compress_v64i8:2997; AVX512VL: # %bb.0:2998; AVX512VL-NEXT: vpsllw $7, %zmm1, %zmm12999; AVX512VL-NEXT: vpmovb2m %zmm1, %k13000; AVX512VL-NEXT: vpcompressb %zmm0, %zmm2 {%k1}3001; AVX512VL-NEXT: vmovdqa64 %zmm2, %zmm03002; AVX512VL-NEXT: retq3003 %out = call <64 x i8> @llvm.experimental.vector.compress(<64 x i8> %vec, <64 x i1> %mask, <64 x i8> %passthru)3004 ret <64 x i8> %out3005}3006 3007define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x i16> %passthru) nounwind {3008; AVX2-LABEL: test_compress_v32i16:3009; AVX2: # %bb.0:3010; AVX2-NEXT: pushq %rbp3011; AVX2-NEXT: movq %rsp, %rbp3012; AVX2-NEXT: pushq %r153013; AVX2-NEXT: pushq %r143014; AVX2-NEXT: pushq %r133015; AVX2-NEXT: pushq %r123016; AVX2-NEXT: pushq %rbx3017; AVX2-NEXT: andq $-32, %rsp3018; AVX2-NEXT: subq $256, %rsp # imm = 0x1003019; AVX2-NEXT: vmovaps %ymm4, {{[0-9]+}}(%rsp)3020; AVX2-NEXT: vmovaps %ymm3, (%rsp)3021; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm33022; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero3023; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]3024; AVX2-NEXT: vpand %ymm5, %ymm4, %ymm43025; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero3026; AVX2-NEXT: vpand %ymm5, %ymm6, %ymm53027; AVX2-NEXT: vpaddw %ymm4, %ymm5, %ymm43028; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm53029; AVX2-NEXT: vpaddw %xmm5, %xmm4, %xmm43030; AVX2-NEXT: vpextrw $1, %xmm4, %eax3031; AVX2-NEXT: vmovd %xmm4, %ecx3032; AVX2-NEXT: addl %eax, %ecx3033; AVX2-NEXT: vpextrw $2, %xmm4, %eax3034; AVX2-NEXT: vpextrw $3, %xmm4, %edx3035; AVX2-NEXT: addl %eax, %edx3036; AVX2-NEXT: addl %ecx, %edx3037; AVX2-NEXT: vpextrw $4, %xmm4, %eax3038; AVX2-NEXT: vpextrw $5, %xmm4, %ecx3039; AVX2-NEXT: addl %eax, %ecx3040; AVX2-NEXT: vpextrw $6, %xmm4, %eax3041; AVX2-NEXT: addl %ecx, %eax3042; AVX2-NEXT: addl %edx, %eax3043; AVX2-NEXT: vpextrw $7, %xmm4, %ecx3044; AVX2-NEXT: addl %eax, %ecx3045; AVX2-NEXT: andl $31, %ecx3046; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3047; AVX2-NEXT: vpextrb $1, %xmm2, %eax3048; AVX2-NEXT: andl $1, %eax3049; AVX2-NEXT: vmovd %xmm2, %ecx3050; AVX2-NEXT: andl $1, %ecx3051; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3052; AVX2-NEXT: addq %rcx, %rax3053; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3054; AVX2-NEXT: vpextrb $2, %xmm2, %ecx3055; AVX2-NEXT: andl $1, %ecx3056; AVX2-NEXT: addq %rax, %rcx3057; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3058; AVX2-NEXT: vpextrb $3, %xmm2, %eax3059; AVX2-NEXT: andl $1, %eax3060; AVX2-NEXT: addq %rcx, %rax3061; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3062; AVX2-NEXT: vpextrb $4, %xmm2, %ecx3063; AVX2-NEXT: andl $1, %ecx3064; AVX2-NEXT: addq %rax, %rcx3065; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3066; AVX2-NEXT: vpextrb $5, %xmm2, %eax3067; AVX2-NEXT: andl $1, %eax3068; AVX2-NEXT: addq %rcx, %rax3069; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3070; AVX2-NEXT: vpextrb $6, %xmm2, %ecx3071; AVX2-NEXT: andl $1, %ecx3072; AVX2-NEXT: addq %rax, %rcx3073; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3074; AVX2-NEXT: vpextrb $7, %xmm2, %eax3075; AVX2-NEXT: andl $1, %eax3076; AVX2-NEXT: addq %rcx, %rax3077; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3078; AVX2-NEXT: vpextrb $8, %xmm2, %ecx3079; AVX2-NEXT: andl $1, %ecx3080; AVX2-NEXT: addq %rax, %rcx3081; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3082; AVX2-NEXT: vpextrb $9, %xmm2, %eax3083; AVX2-NEXT: andl $1, %eax3084; AVX2-NEXT: addq %rcx, %rax3085; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3086; AVX2-NEXT: vpextrb $10, %xmm2, %ecx3087; AVX2-NEXT: andl $1, %ecx3088; AVX2-NEXT: addq %rax, %rcx3089; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3090; AVX2-NEXT: vpextrb $11, %xmm2, %eax3091; AVX2-NEXT: andl $1, %eax3092; AVX2-NEXT: addq %rcx, %rax3093; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3094; AVX2-NEXT: vpextrb $12, %xmm2, %ecx3095; AVX2-NEXT: andl $1, %ecx3096; AVX2-NEXT: addq %rax, %rcx3097; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3098; AVX2-NEXT: vpextrb $13, %xmm2, %eax3099; AVX2-NEXT: andl $1, %eax3100; AVX2-NEXT: addq %rcx, %rax3101; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3102; AVX2-NEXT: vpextrb $14, %xmm2, %ecx3103; AVX2-NEXT: andl $1, %ecx3104; AVX2-NEXT: addq %rax, %rcx3105; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3106; AVX2-NEXT: vpextrb $15, %xmm2, %eax3107; AVX2-NEXT: andl $1, %eax3108; AVX2-NEXT: addq %rcx, %rax3109; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3110; AVX2-NEXT: vmovd %xmm3, %ecx3111; AVX2-NEXT: andl $1, %ecx3112; AVX2-NEXT: addq %rax, %rcx3113; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3114; AVX2-NEXT: vpextrb $1, %xmm3, %eax3115; AVX2-NEXT: andl $1, %eax3116; AVX2-NEXT: addq %rcx, %rax3117; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3118; AVX2-NEXT: vpextrb $2, %xmm3, %ecx3119; AVX2-NEXT: andl $1, %ecx3120; AVX2-NEXT: addq %rax, %rcx3121; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3122; AVX2-NEXT: vpextrb $3, %xmm3, %r12d3123; AVX2-NEXT: andl $1, %r12d3124; AVX2-NEXT: addq %rcx, %r123125; AVX2-NEXT: vpextrb $4, %xmm3, %r15d3126; AVX2-NEXT: andl $1, %r15d3127; AVX2-NEXT: addq %r12, %r153128; AVX2-NEXT: vpextrb $5, %xmm3, %r14d3129; AVX2-NEXT: andl $1, %r14d3130; AVX2-NEXT: addq %r15, %r143131; AVX2-NEXT: vpextrb $6, %xmm3, %ebx3132; AVX2-NEXT: andl $1, %ebx3133; AVX2-NEXT: addq %r14, %rbx3134; AVX2-NEXT: vpextrb $7, %xmm3, %r11d3135; AVX2-NEXT: andl $1, %r11d3136; AVX2-NEXT: addq %rbx, %r113137; AVX2-NEXT: vpextrb $8, %xmm3, %r10d3138; AVX2-NEXT: andl $1, %r10d3139; AVX2-NEXT: addq %r11, %r103140; AVX2-NEXT: vpextrb $9, %xmm3, %r9d3141; AVX2-NEXT: andl $1, %r9d3142; AVX2-NEXT: addq %r10, %r93143; AVX2-NEXT: vpextrb $10, %xmm3, %r8d3144; AVX2-NEXT: andl $1, %r8d3145; AVX2-NEXT: addq %r9, %r83146; AVX2-NEXT: vpextrb $11, %xmm3, %edi3147; AVX2-NEXT: andl $1, %edi3148; AVX2-NEXT: addq %r8, %rdi3149; AVX2-NEXT: vpextrb $12, %xmm3, %esi3150; AVX2-NEXT: andl $1, %esi3151; AVX2-NEXT: addq %rdi, %rsi3152; AVX2-NEXT: vpextrb $13, %xmm3, %edx3153; AVX2-NEXT: andl $1, %edx3154; AVX2-NEXT: addq %rsi, %rdx3155; AVX2-NEXT: vpextrb $14, %xmm3, %ecx3156; AVX2-NEXT: andl $1, %ecx3157; AVX2-NEXT: addq %rdx, %rcx3158; AVX2-NEXT: vpextrb $15, %xmm3, %eax3159; AVX2-NEXT: andl $1, %eax3160; AVX2-NEXT: addq %rcx, %rax3161; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3162; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm23163; AVX2-NEXT: cmpq $32, %rax3164; AVX2-NEXT: vpextrw $7, %xmm2, %eax3165; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3166; AVX2-NEXT: cmovbw (%rsp,%r13,2), %ax3167; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill3168; AVX2-NEXT: vpextrw $0, %xmm0, (%rsp)3169; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3170; AVX2-NEXT: vpextrw $1, %xmm0, (%rsp,%r13,2)3171; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3172; AVX2-NEXT: vpextrw $2, %xmm0, (%rsp,%r13,2)3173; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3174; AVX2-NEXT: vpextrw $3, %xmm0, (%rsp,%r13,2)3175; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3176; AVX2-NEXT: vpextrw $4, %xmm0, (%rsp,%r13,2)3177; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3178; AVX2-NEXT: vpextrw $5, %xmm0, (%rsp,%r13,2)3179; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3180; AVX2-NEXT: andl $31, %r13d3181; AVX2-NEXT: vpextrw $6, %xmm0, (%rsp,%r13,2)3182; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3183; AVX2-NEXT: andl $31, %r13d3184; AVX2-NEXT: vpextrw $7, %xmm0, (%rsp,%r13,2)3185; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3186; AVX2-NEXT: andl $31, %r13d3187; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm03188; AVX2-NEXT: vpextrw $0, %xmm0, (%rsp,%r13,2)3189; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3190; AVX2-NEXT: andl $31, %r13d3191; AVX2-NEXT: vpextrw $1, %xmm0, (%rsp,%r13,2)3192; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3193; AVX2-NEXT: andl $31, %r13d3194; AVX2-NEXT: vpextrw $2, %xmm0, (%rsp,%r13,2)3195; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3196; AVX2-NEXT: andl $31, %r13d3197; AVX2-NEXT: vpextrw $3, %xmm0, (%rsp,%r13,2)3198; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3199; AVX2-NEXT: andl $31, %r13d3200; AVX2-NEXT: vpextrw $4, %xmm0, (%rsp,%r13,2)3201; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3202; AVX2-NEXT: andl $31, %r13d3203; AVX2-NEXT: vpextrw $5, %xmm0, (%rsp,%r13,2)3204; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3205; AVX2-NEXT: andl $31, %r13d3206; AVX2-NEXT: vpextrw $6, %xmm0, (%rsp,%r13,2)3207; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3208; AVX2-NEXT: andl $31, %r13d3209; AVX2-NEXT: vpextrw $7, %xmm0, (%rsp,%r13,2)3210; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3211; AVX2-NEXT: andl $31, %r13d3212; AVX2-NEXT: vpextrw $0, %xmm1, (%rsp,%r13,2)3213; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3214; AVX2-NEXT: andl $31, %r13d3215; AVX2-NEXT: vpextrw $1, %xmm1, (%rsp,%r13,2)3216; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload3217; AVX2-NEXT: andl $31, %r13d3218; AVX2-NEXT: vpextrw $2, %xmm1, (%rsp,%r13,2)3219; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload3220; AVX2-NEXT: andl $31, %eax3221; AVX2-NEXT: vpextrw $3, %xmm1, (%rsp,%rax,2)3222; AVX2-NEXT: andl $31, %r12d3223; AVX2-NEXT: vpextrw $4, %xmm1, (%rsp,%r12,2)3224; AVX2-NEXT: andl $31, %r15d3225; AVX2-NEXT: vpextrw $5, %xmm1, (%rsp,%r15,2)3226; AVX2-NEXT: andl $31, %r14d3227; AVX2-NEXT: vpextrw $6, %xmm1, (%rsp,%r14,2)3228; AVX2-NEXT: andl $31, %ebx3229; AVX2-NEXT: vpextrw $7, %xmm1, (%rsp,%rbx,2)3230; AVX2-NEXT: andl $31, %r11d3231; AVX2-NEXT: vpextrw $0, %xmm2, (%rsp,%r11,2)3232; AVX2-NEXT: andl $31, %r10d3233; AVX2-NEXT: vpextrw $1, %xmm2, (%rsp,%r10,2)3234; AVX2-NEXT: andl $31, %r9d3235; AVX2-NEXT: vpextrw $2, %xmm2, (%rsp,%r9,2)3236; AVX2-NEXT: andl $31, %r8d3237; AVX2-NEXT: vpextrw $3, %xmm2, (%rsp,%r8,2)3238; AVX2-NEXT: andl $31, %edi3239; AVX2-NEXT: vpextrw $4, %xmm2, (%rsp,%rdi,2)3240; AVX2-NEXT: andl $31, %esi3241; AVX2-NEXT: vpextrw $5, %xmm2, (%rsp,%rsi,2)3242; AVX2-NEXT: andl $31, %edx3243; AVX2-NEXT: vpextrw $6, %xmm2, (%rsp,%rdx,2)3244; AVX2-NEXT: andl $31, %ecx3245; AVX2-NEXT: vpextrw $7, %xmm2, (%rsp,%rcx,2)3246; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload3247; AVX2-NEXT: cmpq $31, %rcx3248; AVX2-NEXT: movl $31, %eax3249; AVX2-NEXT: cmovbq %rcx, %rax3250; AVX2-NEXT: movl %eax, %eax3251; AVX2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload3252; AVX2-NEXT: movw %cx, (%rsp,%rax,2)3253; AVX2-NEXT: vmovaps (%rsp), %ymm03254; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm13255; AVX2-NEXT: leaq -40(%rbp), %rsp3256; AVX2-NEXT: popq %rbx3257; AVX2-NEXT: popq %r123258; AVX2-NEXT: popq %r133259; AVX2-NEXT: popq %r143260; AVX2-NEXT: popq %r153261; AVX2-NEXT: popq %rbp3262; AVX2-NEXT: retq3263;3264; AVX512F-LABEL: test_compress_v32i16:3265; AVX512F: # %bb.0:3266; AVX512F-NEXT: pushq %rbp3267; AVX512F-NEXT: movq %rsp, %rbp3268; AVX512F-NEXT: andq $-64, %rsp3269; AVX512F-NEXT: subq $128, %rsp3270; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero3271; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm53272; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero3273; AVX512F-NEXT: vpmovsxbd %xmm5, %zmm53274; AVX512F-NEXT: vpslld $31, %zmm5, %zmm53275; AVX512F-NEXT: vptestmd %zmm5, %zmm5, %k13276; AVX512F-NEXT: vpmovsxbd %xmm1, %zmm13277; AVX512F-NEXT: vpslld $31, %zmm1, %zmm13278; AVX512F-NEXT: vptestmd %zmm1, %zmm1, %k23279; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero3280; AVX512F-NEXT: vpcompressd %zmm1, %zmm1 {%k2} {z}3281; AVX512F-NEXT: vpmovdw %zmm1, (%rsp)3282; AVX512F-NEXT: kshiftrw $8, %k2, %k03283; AVX512F-NEXT: kxorw %k0, %k2, %k03284; AVX512F-NEXT: kshiftrw $4, %k0, %k23285; AVX512F-NEXT: kxorw %k2, %k0, %k03286; AVX512F-NEXT: kshiftrw $2, %k0, %k23287; AVX512F-NEXT: kxorw %k2, %k0, %k03288; AVX512F-NEXT: kshiftrw $1, %k0, %k23289; AVX512F-NEXT: kxorw %k2, %k0, %k03290; AVX512F-NEXT: kmovw %k0, %eax3291; AVX512F-NEXT: andl $31, %eax3292; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm03293; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero3294; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}3295; AVX512F-NEXT: vpmovdw %zmm0, (%rsp,%rax,2)3296; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm03297; AVX512F-NEXT: vpsllw $15, %ymm4, %ymm13298; AVX512F-NEXT: vpsraw $15, %ymm1, %ymm13299; AVX512F-NEXT: vpblendvb %ymm1, {{[0-9]+}}(%rsp), %ymm0, %ymm03300; AVX512F-NEXT: vpsllw $15, %ymm3, %ymm13301; AVX512F-NEXT: vpsraw $15, %ymm1, %ymm13302; AVX512F-NEXT: vpblendvb %ymm1, (%rsp), %ymm2, %ymm13303; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm03304; AVX512F-NEXT: movq %rbp, %rsp3305; AVX512F-NEXT: popq %rbp3306; AVX512F-NEXT: retq3307;3308; AVX512VL-LABEL: test_compress_v32i16:3309; AVX512VL: # %bb.0:3310; AVX512VL-NEXT: vpsllw $7, %ymm1, %ymm13311; AVX512VL-NEXT: vpmovb2m %ymm1, %k13312; AVX512VL-NEXT: vpcompressw %zmm0, %zmm2 {%k1}3313; AVX512VL-NEXT: vmovdqa64 %zmm2, %zmm03314; AVX512VL-NEXT: retq3315 %out = call <32 x i16> @llvm.experimental.vector.compress(<32 x i16> %vec, <32 x i1> %mask, <32 x i16> %passthru)3316 ret <32 x i16> %out3317}3318 3319define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i32> %passthru) nounwind {3320; AVX2-LABEL: test_compress_large:3321; AVX2: # %bb.0:3322; AVX2-NEXT: pushq %rbp3323; AVX2-NEXT: movq %rsp, %rbp3324; AVX2-NEXT: andq $-32, %rsp3325; AVX2-NEXT: subq $288, %rsp # imm = 0x1203326; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3327; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3328; AVX2-NEXT: # kill: def $r9d killed $r9d def $r93329; AVX2-NEXT: # kill: def $r8d killed $r8d def $r83330; AVX2-NEXT: # kill: def $esi killed $esi def $rsi3331; AVX2-NEXT: movq %rdi, %rax3332; AVX2-NEXT: vmovss %xmm0, (%rsp)3333; AVX2-NEXT: andl $1, %esi3334; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rsi,4)3335; AVX2-NEXT: andl $1, %edx3336; AVX2-NEXT: addl %esi, %edx3337; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rdx,4)3338; AVX2-NEXT: andl $1, %ecx3339; AVX2-NEXT: addl %edx, %ecx3340; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rcx,4)3341; AVX2-NEXT: andl $1, %r8d3342; AVX2-NEXT: addl %ecx, %r8d3343; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm03344; AVX2-NEXT: vmovss %xmm0, (%rsp,%r8,4)3345; AVX2-NEXT: andl $1, %r9d3346; AVX2-NEXT: addl %r8d, %r9d3347; AVX2-NEXT: movzbl 16(%rbp), %ecx3348; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%r9,4)3349; AVX2-NEXT: movzbl %cl, %ecx3350; AVX2-NEXT: andl $1, %ecx3351; AVX2-NEXT: addl %r9d, %ecx3352; AVX2-NEXT: movzbl 24(%rbp), %edx3353; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)3354; AVX2-NEXT: movzbl %dl, %edx3355; AVX2-NEXT: andl $1, %edx3356; AVX2-NEXT: addl %ecx, %edx3357; AVX2-NEXT: movzbl 32(%rbp), %ecx3358; AVX2-NEXT: movzbl %cl, %ecx3359; AVX2-NEXT: andl $1, %ecx3360; AVX2-NEXT: addl %edx, %ecx3361; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3362; AVX2-NEXT: andl $63, %edx3363; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rdx,4)3364; AVX2-NEXT: movzbl 40(%rbp), %edx3365; AVX2-NEXT: movzbl %dl, %edx3366; AVX2-NEXT: andl $1, %edx3367; AVX2-NEXT: addl %ecx, %edx3368; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3369; AVX2-NEXT: andl $63, %ecx3370; AVX2-NEXT: vmovss %xmm1, (%rsp,%rcx,4)3371; AVX2-NEXT: movzbl 48(%rbp), %ecx3372; AVX2-NEXT: movzbl %cl, %ecx3373; AVX2-NEXT: andl $1, %ecx3374; AVX2-NEXT: addl %edx, %ecx3375; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3376; AVX2-NEXT: andl $63, %edx3377; AVX2-NEXT: vextractps $1, %xmm1, (%rsp,%rdx,4)3378; AVX2-NEXT: movzbl 56(%rbp), %edx3379; AVX2-NEXT: movzbl %dl, %edx3380; AVX2-NEXT: andl $1, %edx3381; AVX2-NEXT: addl %ecx, %edx3382; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3383; AVX2-NEXT: andl $63, %ecx3384; AVX2-NEXT: vextractps $2, %xmm1, (%rsp,%rcx,4)3385; AVX2-NEXT: movzbl 64(%rbp), %ecx3386; AVX2-NEXT: movzbl %cl, %ecx3387; AVX2-NEXT: andl $1, %ecx3388; AVX2-NEXT: addl %edx, %ecx3389; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3390; AVX2-NEXT: andl $63, %edx3391; AVX2-NEXT: vextractps $3, %xmm1, (%rsp,%rdx,4)3392; AVX2-NEXT: movzbl 72(%rbp), %edx3393; AVX2-NEXT: movzbl %dl, %edx3394; AVX2-NEXT: andl $1, %edx3395; AVX2-NEXT: addl %ecx, %edx3396; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3397; AVX2-NEXT: andl $63, %ecx3398; AVX2-NEXT: vextractf128 $1, %ymm1, %xmm03399; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)3400; AVX2-NEXT: movzbl 80(%rbp), %ecx3401; AVX2-NEXT: movzbl %cl, %ecx3402; AVX2-NEXT: andl $1, %ecx3403; AVX2-NEXT: addl %edx, %ecx3404; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3405; AVX2-NEXT: andl $63, %edx3406; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rdx,4)3407; AVX2-NEXT: movzbl 88(%rbp), %edx3408; AVX2-NEXT: movzbl %dl, %edx3409; AVX2-NEXT: andl $1, %edx3410; AVX2-NEXT: addl %ecx, %edx3411; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3412; AVX2-NEXT: andl $63, %ecx3413; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)3414; AVX2-NEXT: movzbl 96(%rbp), %ecx3415; AVX2-NEXT: movzbl %cl, %ecx3416; AVX2-NEXT: andl $1, %ecx3417; AVX2-NEXT: addl %edx, %ecx3418; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3419; AVX2-NEXT: andl $63, %edx3420; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rdx,4)3421; AVX2-NEXT: movzbl 104(%rbp), %edx3422; AVX2-NEXT: movzbl %dl, %edx3423; AVX2-NEXT: andl $1, %edx3424; AVX2-NEXT: addl %ecx, %edx3425; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3426; AVX2-NEXT: andl $63, %ecx3427; AVX2-NEXT: vmovss %xmm2, (%rsp,%rcx,4)3428; AVX2-NEXT: movzbl 112(%rbp), %ecx3429; AVX2-NEXT: movzbl %cl, %ecx3430; AVX2-NEXT: andl $1, %ecx3431; AVX2-NEXT: addl %edx, %ecx3432; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3433; AVX2-NEXT: andl $63, %edx3434; AVX2-NEXT: vextractps $1, %xmm2, (%rsp,%rdx,4)3435; AVX2-NEXT: movzbl 120(%rbp), %edx3436; AVX2-NEXT: movzbl %dl, %edx3437; AVX2-NEXT: andl $1, %edx3438; AVX2-NEXT: addl %ecx, %edx3439; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3440; AVX2-NEXT: andl $63, %ecx3441; AVX2-NEXT: vextractps $2, %xmm2, (%rsp,%rcx,4)3442; AVX2-NEXT: movzbl 128(%rbp), %ecx3443; AVX2-NEXT: movzbl %cl, %ecx3444; AVX2-NEXT: andl $1, %ecx3445; AVX2-NEXT: addl %edx, %ecx3446; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3447; AVX2-NEXT: andl $63, %edx3448; AVX2-NEXT: vextractps $3, %xmm2, (%rsp,%rdx,4)3449; AVX2-NEXT: movzbl 136(%rbp), %edx3450; AVX2-NEXT: movzbl %dl, %edx3451; AVX2-NEXT: andl $1, %edx3452; AVX2-NEXT: addl %ecx, %edx3453; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3454; AVX2-NEXT: andl $63, %ecx3455; AVX2-NEXT: vextractf128 $1, %ymm2, %xmm03456; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)3457; AVX2-NEXT: movzbl 144(%rbp), %ecx3458; AVX2-NEXT: movzbl %cl, %ecx3459; AVX2-NEXT: andl $1, %ecx3460; AVX2-NEXT: addl %edx, %ecx3461; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3462; AVX2-NEXT: andl $63, %edx3463; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rdx,4)3464; AVX2-NEXT: movzbl 152(%rbp), %edx3465; AVX2-NEXT: movzbl %dl, %edx3466; AVX2-NEXT: andl $1, %edx3467; AVX2-NEXT: addl %ecx, %edx3468; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3469; AVX2-NEXT: andl $63, %ecx3470; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)3471; AVX2-NEXT: movzbl 160(%rbp), %ecx3472; AVX2-NEXT: movzbl %cl, %ecx3473; AVX2-NEXT: andl $1, %ecx3474; AVX2-NEXT: addl %edx, %ecx3475; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3476; AVX2-NEXT: andl $63, %edx3477; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rdx,4)3478; AVX2-NEXT: movzbl 168(%rbp), %edx3479; AVX2-NEXT: movzbl %dl, %edx3480; AVX2-NEXT: andl $1, %edx3481; AVX2-NEXT: addl %ecx, %edx3482; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3483; AVX2-NEXT: andl $63, %ecx3484; AVX2-NEXT: vmovss %xmm3, (%rsp,%rcx,4)3485; AVX2-NEXT: movzbl 176(%rbp), %ecx3486; AVX2-NEXT: movzbl %cl, %ecx3487; AVX2-NEXT: andl $1, %ecx3488; AVX2-NEXT: addl %edx, %ecx3489; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3490; AVX2-NEXT: andl $63, %edx3491; AVX2-NEXT: vextractps $1, %xmm3, (%rsp,%rdx,4)3492; AVX2-NEXT: movzbl 184(%rbp), %edx3493; AVX2-NEXT: movzbl %dl, %edx3494; AVX2-NEXT: andl $1, %edx3495; AVX2-NEXT: addl %ecx, %edx3496; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3497; AVX2-NEXT: andl $63, %ecx3498; AVX2-NEXT: vextractps $2, %xmm3, (%rsp,%rcx,4)3499; AVX2-NEXT: movzbl 192(%rbp), %ecx3500; AVX2-NEXT: movzbl %cl, %ecx3501; AVX2-NEXT: andl $1, %ecx3502; AVX2-NEXT: addl %edx, %ecx3503; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3504; AVX2-NEXT: andl $63, %edx3505; AVX2-NEXT: vextractps $3, %xmm3, (%rsp,%rdx,4)3506; AVX2-NEXT: movzbl 200(%rbp), %edx3507; AVX2-NEXT: movzbl %dl, %edx3508; AVX2-NEXT: andl $1, %edx3509; AVX2-NEXT: addl %ecx, %edx3510; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3511; AVX2-NEXT: andl $63, %ecx3512; AVX2-NEXT: vextractf128 $1, %ymm3, %xmm03513; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)3514; AVX2-NEXT: movzbl 208(%rbp), %ecx3515; AVX2-NEXT: movzbl %cl, %ecx3516; AVX2-NEXT: andl $1, %ecx3517; AVX2-NEXT: addl %edx, %ecx3518; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3519; AVX2-NEXT: andl $63, %edx3520; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rdx,4)3521; AVX2-NEXT: movzbl 216(%rbp), %edx3522; AVX2-NEXT: movzbl %dl, %edx3523; AVX2-NEXT: andl $1, %edx3524; AVX2-NEXT: addl %ecx, %edx3525; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3526; AVX2-NEXT: andl $63, %ecx3527; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)3528; AVX2-NEXT: movzbl 224(%rbp), %ecx3529; AVX2-NEXT: movzbl %cl, %ecx3530; AVX2-NEXT: andl $1, %ecx3531; AVX2-NEXT: addl %edx, %ecx3532; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3533; AVX2-NEXT: andl $63, %edx3534; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rdx,4)3535; AVX2-NEXT: movzbl 232(%rbp), %edx3536; AVX2-NEXT: movzbl %dl, %edx3537; AVX2-NEXT: andl $1, %edx3538; AVX2-NEXT: addl %ecx, %edx3539; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3540; AVX2-NEXT: andl $63, %ecx3541; AVX2-NEXT: vmovss %xmm4, (%rsp,%rcx,4)3542; AVX2-NEXT: movzbl 240(%rbp), %ecx3543; AVX2-NEXT: movzbl %cl, %ecx3544; AVX2-NEXT: andl $1, %ecx3545; AVX2-NEXT: addl %edx, %ecx3546; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3547; AVX2-NEXT: andl $63, %edx3548; AVX2-NEXT: vextractps $1, %xmm4, (%rsp,%rdx,4)3549; AVX2-NEXT: movzbl 248(%rbp), %edx3550; AVX2-NEXT: movzbl %dl, %edx3551; AVX2-NEXT: andl $1, %edx3552; AVX2-NEXT: addl %ecx, %edx3553; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3554; AVX2-NEXT: andl $63, %ecx3555; AVX2-NEXT: vextractps $2, %xmm4, (%rsp,%rcx,4)3556; AVX2-NEXT: movzbl 256(%rbp), %ecx3557; AVX2-NEXT: movzbl %cl, %ecx3558; AVX2-NEXT: andl $1, %ecx3559; AVX2-NEXT: addl %edx, %ecx3560; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3561; AVX2-NEXT: andl $63, %edx3562; AVX2-NEXT: vextractps $3, %xmm4, (%rsp,%rdx,4)3563; AVX2-NEXT: movzbl 264(%rbp), %edx3564; AVX2-NEXT: movzbl %dl, %edx3565; AVX2-NEXT: andl $1, %edx3566; AVX2-NEXT: addl %ecx, %edx3567; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3568; AVX2-NEXT: andl $63, %ecx3569; AVX2-NEXT: vextractf128 $1, %ymm4, %xmm03570; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)3571; AVX2-NEXT: movzbl 272(%rbp), %ecx3572; AVX2-NEXT: movzbl %cl, %ecx3573; AVX2-NEXT: andl $1, %ecx3574; AVX2-NEXT: addl %edx, %ecx3575; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3576; AVX2-NEXT: andl $63, %edx3577; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rdx,4)3578; AVX2-NEXT: movzbl 280(%rbp), %edx3579; AVX2-NEXT: movzbl %dl, %edx3580; AVX2-NEXT: andl $1, %edx3581; AVX2-NEXT: addl %ecx, %edx3582; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3583; AVX2-NEXT: andl $63, %ecx3584; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)3585; AVX2-NEXT: movzbl 288(%rbp), %ecx3586; AVX2-NEXT: movzbl %cl, %ecx3587; AVX2-NEXT: andl $1, %ecx3588; AVX2-NEXT: addl %edx, %ecx3589; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3590; AVX2-NEXT: andl $63, %edx3591; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rdx,4)3592; AVX2-NEXT: movzbl 296(%rbp), %edx3593; AVX2-NEXT: movzbl %dl, %edx3594; AVX2-NEXT: andl $1, %edx3595; AVX2-NEXT: addl %ecx, %edx3596; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3597; AVX2-NEXT: andl $63, %ecx3598; AVX2-NEXT: vmovss %xmm5, (%rsp,%rcx,4)3599; AVX2-NEXT: movzbl 304(%rbp), %ecx3600; AVX2-NEXT: movzbl %cl, %ecx3601; AVX2-NEXT: andl $1, %ecx3602; AVX2-NEXT: addl %edx, %ecx3603; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3604; AVX2-NEXT: andl $63, %edx3605; AVX2-NEXT: vextractps $1, %xmm5, (%rsp,%rdx,4)3606; AVX2-NEXT: movzbl 312(%rbp), %edx3607; AVX2-NEXT: movzbl %dl, %edx3608; AVX2-NEXT: andl $1, %edx3609; AVX2-NEXT: addl %ecx, %edx3610; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3611; AVX2-NEXT: andl $63, %ecx3612; AVX2-NEXT: vextractps $2, %xmm5, (%rsp,%rcx,4)3613; AVX2-NEXT: movzbl 320(%rbp), %ecx3614; AVX2-NEXT: movzbl %cl, %ecx3615; AVX2-NEXT: andl $1, %ecx3616; AVX2-NEXT: addl %edx, %ecx3617; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3618; AVX2-NEXT: andl $63, %edx3619; AVX2-NEXT: vextractps $3, %xmm5, (%rsp,%rdx,4)3620; AVX2-NEXT: movzbl 328(%rbp), %edx3621; AVX2-NEXT: movzbl %dl, %edx3622; AVX2-NEXT: andl $1, %edx3623; AVX2-NEXT: addl %ecx, %edx3624; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3625; AVX2-NEXT: andl $63, %ecx3626; AVX2-NEXT: vextractf128 $1, %ymm5, %xmm03627; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)3628; AVX2-NEXT: movzbl 336(%rbp), %ecx3629; AVX2-NEXT: movzbl %cl, %ecx3630; AVX2-NEXT: andl $1, %ecx3631; AVX2-NEXT: addl %edx, %ecx3632; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3633; AVX2-NEXT: andl $63, %edx3634; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rdx,4)3635; AVX2-NEXT: movzbl 344(%rbp), %edx3636; AVX2-NEXT: movzbl %dl, %edx3637; AVX2-NEXT: andl $1, %edx3638; AVX2-NEXT: addl %ecx, %edx3639; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3640; AVX2-NEXT: andl $63, %ecx3641; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)3642; AVX2-NEXT: movzbl 352(%rbp), %ecx3643; AVX2-NEXT: movzbl %cl, %ecx3644; AVX2-NEXT: andl $1, %ecx3645; AVX2-NEXT: addl %edx, %ecx3646; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3647; AVX2-NEXT: andl $63, %edx3648; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rdx,4)3649; AVX2-NEXT: movzbl 360(%rbp), %edx3650; AVX2-NEXT: movzbl %dl, %edx3651; AVX2-NEXT: andl $1, %edx3652; AVX2-NEXT: addl %ecx, %edx3653; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3654; AVX2-NEXT: andl $63, %ecx3655; AVX2-NEXT: vmovss %xmm6, (%rsp,%rcx,4)3656; AVX2-NEXT: movzbl 368(%rbp), %ecx3657; AVX2-NEXT: movzbl %cl, %ecx3658; AVX2-NEXT: andl $1, %ecx3659; AVX2-NEXT: addl %edx, %ecx3660; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3661; AVX2-NEXT: andl $63, %edx3662; AVX2-NEXT: vextractps $1, %xmm6, (%rsp,%rdx,4)3663; AVX2-NEXT: movzbl 376(%rbp), %edx3664; AVX2-NEXT: movzbl %dl, %edx3665; AVX2-NEXT: andl $1, %edx3666; AVX2-NEXT: addl %ecx, %edx3667; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3668; AVX2-NEXT: andl $63, %ecx3669; AVX2-NEXT: vextractps $2, %xmm6, (%rsp,%rcx,4)3670; AVX2-NEXT: movzbl 384(%rbp), %ecx3671; AVX2-NEXT: movzbl %cl, %ecx3672; AVX2-NEXT: andl $1, %ecx3673; AVX2-NEXT: addl %edx, %ecx3674; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3675; AVX2-NEXT: andl $63, %edx3676; AVX2-NEXT: vextractps $3, %xmm6, (%rsp,%rdx,4)3677; AVX2-NEXT: movzbl 392(%rbp), %edx3678; AVX2-NEXT: movzbl %dl, %edx3679; AVX2-NEXT: andl $1, %edx3680; AVX2-NEXT: addl %ecx, %edx3681; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3682; AVX2-NEXT: andl $63, %ecx3683; AVX2-NEXT: vextractf128 $1, %ymm6, %xmm03684; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)3685; AVX2-NEXT: movzbl 400(%rbp), %ecx3686; AVX2-NEXT: movzbl %cl, %ecx3687; AVX2-NEXT: andl $1, %ecx3688; AVX2-NEXT: addl %edx, %ecx3689; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3690; AVX2-NEXT: andl $63, %edx3691; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rdx,4)3692; AVX2-NEXT: movzbl 408(%rbp), %edx3693; AVX2-NEXT: movzbl %dl, %edx3694; AVX2-NEXT: andl $1, %edx3695; AVX2-NEXT: addl %ecx, %edx3696; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3697; AVX2-NEXT: andl $63, %ecx3698; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)3699; AVX2-NEXT: movzbl 416(%rbp), %ecx3700; AVX2-NEXT: movzbl %cl, %ecx3701; AVX2-NEXT: andl $1, %ecx3702; AVX2-NEXT: addl %edx, %ecx3703; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3704; AVX2-NEXT: andl $63, %edx3705; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rdx,4)3706; AVX2-NEXT: movzbl 424(%rbp), %edx3707; AVX2-NEXT: movzbl %dl, %edx3708; AVX2-NEXT: andl $1, %edx3709; AVX2-NEXT: addl %ecx, %edx3710; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3711; AVX2-NEXT: andl $63, %ecx3712; AVX2-NEXT: vmovss %xmm7, (%rsp,%rcx,4)3713; AVX2-NEXT: movzbl 432(%rbp), %ecx3714; AVX2-NEXT: movzbl %cl, %ecx3715; AVX2-NEXT: andl $1, %ecx3716; AVX2-NEXT: addl %edx, %ecx3717; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3718; AVX2-NEXT: andl $63, %edx3719; AVX2-NEXT: vextractps $1, %xmm7, (%rsp,%rdx,4)3720; AVX2-NEXT: movzbl 440(%rbp), %edx3721; AVX2-NEXT: movzbl %dl, %edx3722; AVX2-NEXT: andl $1, %edx3723; AVX2-NEXT: addl %ecx, %edx3724; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3725; AVX2-NEXT: andl $63, %ecx3726; AVX2-NEXT: vextractps $2, %xmm7, (%rsp,%rcx,4)3727; AVX2-NEXT: movzbl 448(%rbp), %ecx3728; AVX2-NEXT: movzbl %cl, %ecx3729; AVX2-NEXT: andl $1, %ecx3730; AVX2-NEXT: addl %edx, %ecx3731; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3732; AVX2-NEXT: andl $63, %edx3733; AVX2-NEXT: vextractps $3, %xmm7, (%rsp,%rdx,4)3734; AVX2-NEXT: movzbl 456(%rbp), %edx3735; AVX2-NEXT: movzbl %dl, %edx3736; AVX2-NEXT: andl $1, %edx3737; AVX2-NEXT: addl %ecx, %edx3738; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3739; AVX2-NEXT: andl $63, %ecx3740; AVX2-NEXT: vextractf128 $1, %ymm7, %xmm03741; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)3742; AVX2-NEXT: movzbl 464(%rbp), %ecx3743; AVX2-NEXT: movzbl %cl, %ecx3744; AVX2-NEXT: andl $1, %ecx3745; AVX2-NEXT: addl %edx, %ecx3746; AVX2-NEXT: # kill: def $edx killed $edx def $rdx3747; AVX2-NEXT: andl $63, %edx3748; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rdx,4)3749; AVX2-NEXT: movzbl 472(%rbp), %edx3750; AVX2-NEXT: movzbl %dl, %edx3751; AVX2-NEXT: andl $1, %edx3752; AVX2-NEXT: addl %ecx, %edx3753; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx3754; AVX2-NEXT: andl $63, %ecx3755; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)3756; AVX2-NEXT: andl $63, %edx3757; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rdx,4)3758; AVX2-NEXT: vmovaps (%rsp), %ymm03759; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm13760; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm23761; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm33762; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm43763; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm53764; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm63765; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm73766; AVX2-NEXT: vmovaps %ymm7, 224(%rdi)3767; AVX2-NEXT: vmovaps %ymm6, 192(%rdi)3768; AVX2-NEXT: vmovaps %ymm5, 160(%rdi)3769; AVX2-NEXT: vmovaps %ymm4, 128(%rdi)3770; AVX2-NEXT: vmovaps %ymm3, 96(%rdi)3771; AVX2-NEXT: vmovaps %ymm2, 64(%rdi)3772; AVX2-NEXT: vmovaps %ymm1, 32(%rdi)3773; AVX2-NEXT: vmovaps %ymm0, (%rdi)3774; AVX2-NEXT: movq %rbp, %rsp3775; AVX2-NEXT: popq %rbp3776; AVX2-NEXT: vzeroupper3777; AVX2-NEXT: retq3778;3779; AVX512F-LABEL: test_compress_large:3780; AVX512F: # %bb.0:3781; AVX512F-NEXT: pushq %rbp3782; AVX512F-NEXT: movq %rsp, %rbp3783; AVX512F-NEXT: andq $-64, %rsp3784; AVX512F-NEXT: subq $640, %rsp # imm = 0x2803785; AVX512F-NEXT: movzbl 352(%rbp), %eax3786; AVX512F-NEXT: andl $1, %eax3787; AVX512F-NEXT: kmovw %eax, %k03788; AVX512F-NEXT: movzbl 360(%rbp), %eax3789; AVX512F-NEXT: kmovw %eax, %k13790; AVX512F-NEXT: kshiftlw $15, %k1, %k13791; AVX512F-NEXT: kshiftrw $14, %k1, %k13792; AVX512F-NEXT: korw %k1, %k0, %k03793; AVX512F-NEXT: movw $-5, %ax3794; AVX512F-NEXT: kmovw %eax, %k13795; AVX512F-NEXT: kandw %k1, %k0, %k03796; AVX512F-NEXT: kmovw %k1, %k33797; AVX512F-NEXT: movzbl 368(%rbp), %eax3798; AVX512F-NEXT: kmovw %eax, %k13799; AVX512F-NEXT: kshiftlw $15, %k1, %k13800; AVX512F-NEXT: kshiftrw $13, %k1, %k13801; AVX512F-NEXT: korw %k1, %k0, %k03802; AVX512F-NEXT: movw $-9, %ax3803; AVX512F-NEXT: kmovw %eax, %k53804; AVX512F-NEXT: kandw %k5, %k0, %k03805; AVX512F-NEXT: movzbl 376(%rbp), %eax3806; AVX512F-NEXT: kmovw %eax, %k13807; AVX512F-NEXT: kshiftlw $15, %k1, %k13808; AVX512F-NEXT: kshiftrw $12, %k1, %k13809; AVX512F-NEXT: korw %k1, %k0, %k03810; AVX512F-NEXT: movw $-17, %ax3811; AVX512F-NEXT: kmovw %eax, %k63812; AVX512F-NEXT: kandw %k6, %k0, %k03813; AVX512F-NEXT: movzbl 384(%rbp), %eax3814; AVX512F-NEXT: kmovw %eax, %k13815; AVX512F-NEXT: kshiftlw $15, %k1, %k13816; AVX512F-NEXT: kshiftrw $11, %k1, %k13817; AVX512F-NEXT: korw %k1, %k0, %k03818; AVX512F-NEXT: movw $-33, %ax3819; AVX512F-NEXT: kmovw %eax, %k73820; AVX512F-NEXT: kandw %k7, %k0, %k03821; AVX512F-NEXT: movzbl 392(%rbp), %eax3822; AVX512F-NEXT: kmovw %eax, %k13823; AVX512F-NEXT: kshiftlw $15, %k1, %k13824; AVX512F-NEXT: kshiftrw $10, %k1, %k13825; AVX512F-NEXT: korw %k1, %k0, %k03826; AVX512F-NEXT: movw $-65, %ax3827; AVX512F-NEXT: kmovw %eax, %k13828; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill3829; AVX512F-NEXT: kandw %k1, %k0, %k03830; AVX512F-NEXT: movzbl 400(%rbp), %eax3831; AVX512F-NEXT: kmovw %eax, %k13832; AVX512F-NEXT: kshiftlw $15, %k1, %k13833; AVX512F-NEXT: kshiftrw $9, %k1, %k13834; AVX512F-NEXT: korw %k1, %k0, %k03835; AVX512F-NEXT: movw $-129, %ax3836; AVX512F-NEXT: kmovw %eax, %k13837; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill3838; AVX512F-NEXT: kandw %k1, %k0, %k03839; AVX512F-NEXT: movzbl 408(%rbp), %eax3840; AVX512F-NEXT: kmovw %eax, %k13841; AVX512F-NEXT: kshiftlw $15, %k1, %k13842; AVX512F-NEXT: kshiftrw $8, %k1, %k13843; AVX512F-NEXT: korw %k1, %k0, %k03844; AVX512F-NEXT: movw $-257, %ax # imm = 0xFEFF3845; AVX512F-NEXT: kmovw %eax, %k13846; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill3847; AVX512F-NEXT: kandw %k1, %k0, %k03848; AVX512F-NEXT: movzbl 416(%rbp), %eax3849; AVX512F-NEXT: kmovw %eax, %k13850; AVX512F-NEXT: kshiftlw $15, %k1, %k13851; AVX512F-NEXT: kshiftrw $7, %k1, %k13852; AVX512F-NEXT: korw %k1, %k0, %k03853; AVX512F-NEXT: movw $-513, %ax # imm = 0xFDFF3854; AVX512F-NEXT: kmovw %eax, %k13855; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill3856; AVX512F-NEXT: kandw %k1, %k0, %k03857; AVX512F-NEXT: movzbl 424(%rbp), %eax3858; AVX512F-NEXT: kmovw %eax, %k13859; AVX512F-NEXT: kshiftlw $15, %k1, %k13860; AVX512F-NEXT: kshiftrw $6, %k1, %k13861; AVX512F-NEXT: korw %k1, %k0, %k03862; AVX512F-NEXT: movw $-1025, %ax # imm = 0xFBFF3863; AVX512F-NEXT: kmovw %eax, %k13864; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill3865; AVX512F-NEXT: kandw %k1, %k0, %k03866; AVX512F-NEXT: movzbl 432(%rbp), %eax3867; AVX512F-NEXT: kmovw %eax, %k13868; AVX512F-NEXT: kshiftlw $15, %k1, %k13869; AVX512F-NEXT: kshiftrw $5, %k1, %k13870; AVX512F-NEXT: korw %k1, %k0, %k03871; AVX512F-NEXT: movw $-2049, %ax # imm = 0xF7FF3872; AVX512F-NEXT: kmovw %eax, %k13873; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill3874; AVX512F-NEXT: kandw %k1, %k0, %k03875; AVX512F-NEXT: movzbl 440(%rbp), %eax3876; AVX512F-NEXT: kmovw %eax, %k13877; AVX512F-NEXT: kshiftlw $15, %k1, %k13878; AVX512F-NEXT: kshiftrw $4, %k1, %k13879; AVX512F-NEXT: korw %k1, %k0, %k03880; AVX512F-NEXT: movw $-4097, %ax # imm = 0xEFFF3881; AVX512F-NEXT: kmovw %eax, %k13882; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill3883; AVX512F-NEXT: kandw %k1, %k0, %k03884; AVX512F-NEXT: movzbl 448(%rbp), %eax3885; AVX512F-NEXT: kmovw %eax, %k13886; AVX512F-NEXT: kshiftlw $15, %k1, %k13887; AVX512F-NEXT: kshiftrw $3, %k1, %k13888; AVX512F-NEXT: korw %k1, %k0, %k03889; AVX512F-NEXT: movw $-8193, %ax # imm = 0xDFFF3890; AVX512F-NEXT: kmovw %eax, %k13891; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill3892; AVX512F-NEXT: kandw %k1, %k0, %k03893; AVX512F-NEXT: movzbl 456(%rbp), %eax3894; AVX512F-NEXT: kmovw %eax, %k13895; AVX512F-NEXT: kshiftlw $15, %k1, %k13896; AVX512F-NEXT: kshiftrw $2, %k1, %k13897; AVX512F-NEXT: korw %k1, %k0, %k13898; AVX512F-NEXT: movw $-16385, %ax # imm = 0xBFFF3899; AVX512F-NEXT: kmovw %eax, %k43900; AVX512F-NEXT: kandw %k4, %k1, %k13901; AVX512F-NEXT: kmovw %k4, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill3902; AVX512F-NEXT: movzbl 464(%rbp), %eax3903; AVX512F-NEXT: kmovw %eax, %k23904; AVX512F-NEXT: kshiftlw $14, %k2, %k23905; AVX512F-NEXT: korw %k2, %k1, %k13906; AVX512F-NEXT: kshiftlw $1, %k1, %k13907; AVX512F-NEXT: kshiftrw $1, %k1, %k13908; AVX512F-NEXT: movzbl 472(%rbp), %eax3909; AVX512F-NEXT: kmovw %eax, %k23910; AVX512F-NEXT: kshiftlw $15, %k2, %k23911; AVX512F-NEXT: korw %k2, %k1, %k13912; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill3913; AVX512F-NEXT: movzbl 224(%rbp), %eax3914; AVX512F-NEXT: andl $1, %eax3915; AVX512F-NEXT: movzbl 232(%rbp), %r10d3916; AVX512F-NEXT: kmovw %r10d, %k13917; AVX512F-NEXT: kshiftlw $15, %k1, %k13918; AVX512F-NEXT: kshiftrw $14, %k1, %k13919; AVX512F-NEXT: kmovw %eax, %k23920; AVX512F-NEXT: korw %k1, %k2, %k13921; AVX512F-NEXT: kmovw %k3, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill3922; AVX512F-NEXT: kandw %k3, %k1, %k13923; AVX512F-NEXT: movzbl 240(%rbp), %eax3924; AVX512F-NEXT: kmovw %eax, %k23925; AVX512F-NEXT: kshiftlw $15, %k2, %k23926; AVX512F-NEXT: kshiftrw $13, %k2, %k23927; AVX512F-NEXT: korw %k2, %k1, %k13928; AVX512F-NEXT: kandw %k5, %k1, %k13929; AVX512F-NEXT: movzbl 248(%rbp), %eax3930; AVX512F-NEXT: kmovw %eax, %k23931; AVX512F-NEXT: kshiftlw $15, %k2, %k23932; AVX512F-NEXT: kshiftrw $12, %k2, %k23933; AVX512F-NEXT: korw %k2, %k1, %k13934; AVX512F-NEXT: kandw %k6, %k1, %k13935; AVX512F-NEXT: movzbl 256(%rbp), %eax3936; AVX512F-NEXT: kmovw %eax, %k23937; AVX512F-NEXT: kshiftlw $15, %k2, %k23938; AVX512F-NEXT: kshiftrw $11, %k2, %k23939; AVX512F-NEXT: korw %k2, %k1, %k13940; AVX512F-NEXT: kandw %k7, %k1, %k13941; AVX512F-NEXT: movzbl 264(%rbp), %eax3942; AVX512F-NEXT: kmovw %eax, %k23943; AVX512F-NEXT: kshiftlw $15, %k2, %k23944; AVX512F-NEXT: kshiftrw $10, %k2, %k23945; AVX512F-NEXT: korw %k2, %k1, %k13946; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload3947; AVX512F-NEXT: kandw %k0, %k1, %k13948; AVX512F-NEXT: movzbl 272(%rbp), %eax3949; AVX512F-NEXT: kmovw %eax, %k23950; AVX512F-NEXT: kshiftlw $15, %k2, %k23951; AVX512F-NEXT: kshiftrw $9, %k2, %k23952; AVX512F-NEXT: korw %k2, %k1, %k03953; AVX512F-NEXT: kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill3954; AVX512F-NEXT: movzbl 280(%rbp), %eax3955; AVX512F-NEXT: kmovw %eax, %k13956; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill3957; AVX512F-NEXT: kshiftlw $15, %k1, %k13958; AVX512F-NEXT: kshiftrw $8, %k1, %k13959; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload3960; AVX512F-NEXT: kandw %k2, %k0, %k23961; AVX512F-NEXT: korw %k1, %k2, %k13962; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload3963; AVX512F-NEXT: kandw %k0, %k1, %k13964; AVX512F-NEXT: movzbl 288(%rbp), %eax3965; AVX512F-NEXT: kmovw %eax, %k03966; AVX512F-NEXT: kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill3967; AVX512F-NEXT: kshiftlw $15, %k0, %k23968; AVX512F-NEXT: kshiftrw $7, %k2, %k23969; AVX512F-NEXT: korw %k2, %k1, %k13970; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload3971; AVX512F-NEXT: kandw %k0, %k1, %k13972; AVX512F-NEXT: movzbl 296(%rbp), %eax3973; AVX512F-NEXT: kmovw %eax, %k23974; AVX512F-NEXT: kshiftlw $15, %k2, %k03975; AVX512F-NEXT: kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill3976; AVX512F-NEXT: kshiftrw $6, %k0, %k23977; AVX512F-NEXT: korw %k2, %k1, %k13978; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload3979; AVX512F-NEXT: kandw %k0, %k1, %k13980; AVX512F-NEXT: movzbl 304(%rbp), %eax3981; AVX512F-NEXT: kmovw %eax, %k23982; AVX512F-NEXT: kshiftlw $15, %k2, %k03983; AVX512F-NEXT: kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill3984; AVX512F-NEXT: kshiftrw $5, %k0, %k23985; AVX512F-NEXT: korw %k2, %k1, %k13986; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload3987; AVX512F-NEXT: kandw %k0, %k1, %k13988; AVX512F-NEXT: movzbl 312(%rbp), %eax3989; AVX512F-NEXT: kmovw %eax, %k23990; AVX512F-NEXT: kshiftlw $15, %k2, %k03991; AVX512F-NEXT: kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill3992; AVX512F-NEXT: kshiftrw $4, %k0, %k23993; AVX512F-NEXT: korw %k2, %k1, %k13994; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload3995; AVX512F-NEXT: kandw %k0, %k1, %k13996; AVX512F-NEXT: movzbl 320(%rbp), %eax3997; AVX512F-NEXT: kmovw %eax, %k23998; AVX512F-NEXT: kshiftlw $15, %k2, %k03999; AVX512F-NEXT: kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill4000; AVX512F-NEXT: kshiftrw $3, %k0, %k24001; AVX512F-NEXT: korw %k2, %k1, %k14002; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload4003; AVX512F-NEXT: kandw %k0, %k1, %k14004; AVX512F-NEXT: movzbl 328(%rbp), %eax4005; AVX512F-NEXT: kmovw %eax, %k24006; AVX512F-NEXT: kshiftlw $15, %k2, %k24007; AVX512F-NEXT: kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill4008; AVX512F-NEXT: kshiftrw $2, %k2, %k24009; AVX512F-NEXT: korw %k2, %k1, %k14010; AVX512F-NEXT: kandw %k4, %k1, %k14011; AVX512F-NEXT: movzbl 336(%rbp), %eax4012; AVX512F-NEXT: kmovw %eax, %k24013; AVX512F-NEXT: kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill4014; AVX512F-NEXT: kshiftlw $14, %k2, %k24015; AVX512F-NEXT: korw %k2, %k1, %k14016; AVX512F-NEXT: kshiftlw $1, %k1, %k14017; AVX512F-NEXT: kshiftrw $1, %k1, %k14018; AVX512F-NEXT: movzbl 344(%rbp), %eax4019; AVX512F-NEXT: kmovw %eax, %k24020; AVX512F-NEXT: kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill4021; AVX512F-NEXT: kshiftlw $15, %k2, %k24022; AVX512F-NEXT: korw %k2, %k1, %k14023; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill4024; AVX512F-NEXT: movzbl 96(%rbp), %eax4025; AVX512F-NEXT: andl $1, %eax4026; AVX512F-NEXT: movzbl 104(%rbp), %r10d4027; AVX512F-NEXT: kmovw %r10d, %k14028; AVX512F-NEXT: kshiftlw $15, %k1, %k14029; AVX512F-NEXT: kshiftrw $14, %k1, %k14030; AVX512F-NEXT: kmovw %eax, %k24031; AVX512F-NEXT: korw %k1, %k2, %k14032; AVX512F-NEXT: kandw %k3, %k1, %k14033; AVX512F-NEXT: movzbl 112(%rbp), %eax4034; AVX512F-NEXT: kmovw %eax, %k24035; AVX512F-NEXT: kshiftlw $15, %k2, %k24036; AVX512F-NEXT: kshiftrw $13, %k2, %k24037; AVX512F-NEXT: korw %k2, %k1, %k14038; AVX512F-NEXT: kmovw %k5, %k44039; AVX512F-NEXT: kmovw %k5, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill4040; AVX512F-NEXT: kandw %k5, %k1, %k14041; AVX512F-NEXT: movzbl 120(%rbp), %eax4042; AVX512F-NEXT: kmovw %eax, %k24043; AVX512F-NEXT: kshiftlw $15, %k2, %k24044; AVX512F-NEXT: kshiftrw $12, %k2, %k24045; AVX512F-NEXT: korw %k2, %k1, %k14046; AVX512F-NEXT: kmovw %k6, %k54047; AVX512F-NEXT: kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill4048; AVX512F-NEXT: kandw %k6, %k1, %k14049; AVX512F-NEXT: movzbl 128(%rbp), %eax4050; AVX512F-NEXT: kmovw %eax, %k24051; AVX512F-NEXT: kshiftlw $15, %k2, %k24052; AVX512F-NEXT: kshiftrw $11, %k2, %k24053; AVX512F-NEXT: korw %k2, %k1, %k14054; AVX512F-NEXT: kmovw %k7, %k64055; AVX512F-NEXT: kmovw %k7, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill4056; AVX512F-NEXT: kandw %k7, %k1, %k14057; AVX512F-NEXT: movzbl 136(%rbp), %eax4058; AVX512F-NEXT: kmovw %eax, %k24059; AVX512F-NEXT: kshiftlw $15, %k2, %k24060; AVX512F-NEXT: kshiftrw $10, %k2, %k24061; AVX512F-NEXT: korw %k2, %k1, %k14062; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload4063; AVX512F-NEXT: kandw %k7, %k1, %k14064; AVX512F-NEXT: movzbl 144(%rbp), %eax4065; AVX512F-NEXT: kmovw %eax, %k24066; AVX512F-NEXT: kshiftlw $15, %k2, %k24067; AVX512F-NEXT: kshiftrw $9, %k2, %k24068; AVX512F-NEXT: korw %k2, %k1, %k14069; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload4070; AVX512F-NEXT: kandw %k2, %k1, %k14071; AVX512F-NEXT: movzbl 152(%rbp), %eax4072; AVX512F-NEXT: kmovw %eax, %k24073; AVX512F-NEXT: kshiftlw $15, %k2, %k24074; AVX512F-NEXT: kshiftrw $8, %k2, %k24075; AVX512F-NEXT: korw %k2, %k1, %k14076; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload4077; AVX512F-NEXT: kandw %k3, %k1, %k14078; AVX512F-NEXT: movzbl 160(%rbp), %eax4079; AVX512F-NEXT: kmovw %eax, %k24080; AVX512F-NEXT: kshiftlw $15, %k2, %k24081; AVX512F-NEXT: kshiftrw $7, %k2, %k24082; AVX512F-NEXT: korw %k2, %k1, %k14083; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload4084; AVX512F-NEXT: kandw %k2, %k1, %k14085; AVX512F-NEXT: movzbl 168(%rbp), %eax4086; AVX512F-NEXT: kmovw %eax, %k24087; AVX512F-NEXT: kshiftlw $15, %k2, %k24088; AVX512F-NEXT: kshiftrw $6, %k2, %k24089; AVX512F-NEXT: korw %k2, %k1, %k14090; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload4091; AVX512F-NEXT: kandw %k2, %k1, %k14092; AVX512F-NEXT: movzbl 176(%rbp), %eax4093; AVX512F-NEXT: kmovw %eax, %k24094; AVX512F-NEXT: kshiftlw $15, %k2, %k24095; AVX512F-NEXT: kshiftrw $5, %k2, %k24096; AVX512F-NEXT: korw %k2, %k1, %k14097; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload4098; AVX512F-NEXT: kandw %k2, %k1, %k14099; AVX512F-NEXT: movzbl 184(%rbp), %eax4100; AVX512F-NEXT: kmovw %eax, %k24101; AVX512F-NEXT: kshiftlw $15, %k2, %k24102; AVX512F-NEXT: kshiftrw $4, %k2, %k24103; AVX512F-NEXT: korw %k2, %k1, %k14104; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload4105; AVX512F-NEXT: kandw %k2, %k1, %k14106; AVX512F-NEXT: movzbl 192(%rbp), %eax4107; AVX512F-NEXT: kmovw %eax, %k24108; AVX512F-NEXT: kshiftlw $15, %k2, %k24109; AVX512F-NEXT: kshiftrw $3, %k2, %k24110; AVX512F-NEXT: korw %k2, %k1, %k14111; AVX512F-NEXT: kandw %k0, %k1, %k14112; AVX512F-NEXT: movzbl 200(%rbp), %eax4113; AVX512F-NEXT: kmovw %eax, %k24114; AVX512F-NEXT: kshiftlw $15, %k2, %k24115; AVX512F-NEXT: kshiftrw $2, %k2, %k24116; AVX512F-NEXT: korw %k2, %k1, %k14117; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload4118; AVX512F-NEXT: kandw %k2, %k1, %k14119; AVX512F-NEXT: movzbl 208(%rbp), %eax4120; AVX512F-NEXT: kmovw %eax, %k24121; AVX512F-NEXT: kshiftlw $14, %k2, %k24122; AVX512F-NEXT: korw %k2, %k1, %k14123; AVX512F-NEXT: kshiftlw $1, %k1, %k14124; AVX512F-NEXT: kshiftrw $1, %k1, %k14125; AVX512F-NEXT: movzbl 216(%rbp), %eax4126; AVX512F-NEXT: kmovw %eax, %k24127; AVX512F-NEXT: kshiftlw $15, %k2, %k24128; AVX512F-NEXT: korw %k2, %k1, %k14129; AVX512F-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill4130; AVX512F-NEXT: andl $1, %edi4131; AVX512F-NEXT: kmovw %esi, %k14132; AVX512F-NEXT: kshiftlw $15, %k1, %k14133; AVX512F-NEXT: kshiftrw $14, %k1, %k14134; AVX512F-NEXT: kmovw %edi, %k24135; AVX512F-NEXT: korw %k1, %k2, %k14136; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload4137; AVX512F-NEXT: kandw %k2, %k1, %k14138; AVX512F-NEXT: kmovw %edx, %k24139; AVX512F-NEXT: kshiftlw $15, %k2, %k24140; AVX512F-NEXT: kshiftrw $13, %k2, %k24141; AVX512F-NEXT: korw %k2, %k1, %k14142; AVX512F-NEXT: kandw %k4, %k1, %k14143; AVX512F-NEXT: kmovw %ecx, %k24144; AVX512F-NEXT: kshiftlw $15, %k2, %k24145; AVX512F-NEXT: kshiftrw $12, %k2, %k24146; AVX512F-NEXT: korw %k2, %k1, %k14147; AVX512F-NEXT: kandw %k5, %k1, %k14148; AVX512F-NEXT: kmovw %r8d, %k24149; AVX512F-NEXT: kshiftlw $15, %k2, %k24150; AVX512F-NEXT: kshiftrw $11, %k2, %k24151; AVX512F-NEXT: korw %k2, %k1, %k14152; AVX512F-NEXT: kandw %k6, %k1, %k14153; AVX512F-NEXT: kmovw %r9d, %k24154; AVX512F-NEXT: kshiftlw $15, %k2, %k24155; AVX512F-NEXT: kshiftrw $10, %k2, %k24156; AVX512F-NEXT: korw %k2, %k1, %k14157; AVX512F-NEXT: kandw %k7, %k1, %k14158; AVX512F-NEXT: movzbl 16(%rbp), %eax4159; AVX512F-NEXT: kmovw %eax, %k24160; AVX512F-NEXT: kshiftlw $15, %k2, %k24161; AVX512F-NEXT: kshiftrw $9, %k2, %k24162; AVX512F-NEXT: korw %k2, %k1, %k24163; AVX512F-NEXT: kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill4164; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload4165; AVX512F-NEXT: kandw %k1, %k2, %k14166; AVX512F-NEXT: movzbl 24(%rbp), %eax4167; AVX512F-NEXT: kmovw %eax, %k24168; AVX512F-NEXT: kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill4169; AVX512F-NEXT: kshiftlw $15, %k2, %k24170; AVX512F-NEXT: kshiftrw $8, %k2, %k24171; AVX512F-NEXT: korw %k2, %k1, %k14172; AVX512F-NEXT: kandw %k3, %k1, %k14173; AVX512F-NEXT: movzbl 32(%rbp), %eax4174; AVX512F-NEXT: kmovw %eax, %k24175; AVX512F-NEXT: kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill4176; AVX512F-NEXT: kshiftlw $15, %k2, %k24177; AVX512F-NEXT: kshiftrw $7, %k2, %k24178; AVX512F-NEXT: korw %k2, %k1, %k14179; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload4180; AVX512F-NEXT: kandw %k2, %k1, %k14181; AVX512F-NEXT: movzbl 40(%rbp), %eax4182; AVX512F-NEXT: kmovw %eax, %k24183; AVX512F-NEXT: kshiftlw $15, %k2, %k24184; AVX512F-NEXT: kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill4185; AVX512F-NEXT: kshiftrw $6, %k2, %k24186; AVX512F-NEXT: korw %k2, %k1, %k14187; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload4188; AVX512F-NEXT: kandw %k2, %k1, %k14189; AVX512F-NEXT: movzbl 48(%rbp), %eax4190; AVX512F-NEXT: kmovw %eax, %k24191; AVX512F-NEXT: kshiftlw $15, %k2, %k54192; AVX512F-NEXT: kshiftrw $5, %k5, %k24193; AVX512F-NEXT: korw %k2, %k1, %k14194; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload4195; AVX512F-NEXT: kandw %k2, %k1, %k14196; AVX512F-NEXT: movzbl 56(%rbp), %eax4197; AVX512F-NEXT: kmovw %eax, %k24198; AVX512F-NEXT: kshiftlw $15, %k2, %k44199; AVX512F-NEXT: kshiftrw $4, %k4, %k24200; AVX512F-NEXT: korw %k2, %k1, %k14201; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload4202; AVX512F-NEXT: kandw %k2, %k1, %k14203; AVX512F-NEXT: movzbl 64(%rbp), %eax4204; AVX512F-NEXT: kmovw %eax, %k24205; AVX512F-NEXT: kshiftlw $15, %k2, %k34206; AVX512F-NEXT: kshiftrw $3, %k3, %k24207; AVX512F-NEXT: korw %k2, %k1, %k14208; AVX512F-NEXT: kandw %k0, %k1, %k14209; AVX512F-NEXT: movzbl 72(%rbp), %eax4210; AVX512F-NEXT: kmovw %eax, %k24211; AVX512F-NEXT: kshiftlw $15, %k2, %k24212; AVX512F-NEXT: kshiftrw $2, %k2, %k04213; AVX512F-NEXT: korw %k0, %k1, %k04214; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload4215; AVX512F-NEXT: kandw %k1, %k0, %k04216; AVX512F-NEXT: movzbl 80(%rbp), %eax4217; AVX512F-NEXT: kmovw %eax, %k14218; AVX512F-NEXT: kshiftlw $14, %k1, %k74219; AVX512F-NEXT: korw %k7, %k0, %k04220; AVX512F-NEXT: kshiftlw $1, %k0, %k04221; AVX512F-NEXT: kshiftrw $1, %k0, %k74222; AVX512F-NEXT: movzbl 88(%rbp), %eax4223; AVX512F-NEXT: kmovw %eax, %k04224; AVX512F-NEXT: kshiftlw $15, %k0, %k64225; AVX512F-NEXT: korw %k6, %k7, %k64226; AVX512F-NEXT: kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill4227; AVX512F-NEXT: movw $-3, %ax4228; AVX512F-NEXT: kmovw %eax, %k64229; AVX512F-NEXT: kmovw %k6, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill4230; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload4231; AVX512F-NEXT: kandw %k6, %k7, %k64232; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload4233; AVX512F-NEXT: kshiftrw $14, %k7, %k74234; AVX512F-NEXT: korw %k7, %k6, %k64235; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload4236; AVX512F-NEXT: kandw %k7, %k6, %k64237; AVX512F-NEXT: kshiftrw $13, %k5, %k54238; AVX512F-NEXT: korw %k5, %k6, %k54239; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k6 # 2-byte Reload4240; AVX512F-NEXT: kandw %k6, %k5, %k54241; AVX512F-NEXT: kshiftrw $12, %k4, %k44242; AVX512F-NEXT: korw %k4, %k5, %k44243; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload4244; AVX512F-NEXT: kandw %k5, %k4, %k44245; AVX512F-NEXT: kshiftrw $11, %k3, %k34246; AVX512F-NEXT: korw %k3, %k4, %k34247; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 # 2-byte Reload4248; AVX512F-NEXT: kandw %k4, %k3, %k34249; AVX512F-NEXT: kshiftrw $10, %k2, %k24250; AVX512F-NEXT: korw %k2, %k3, %k24251; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload4252; AVX512F-NEXT: kandw %k3, %k2, %k24253; AVX512F-NEXT: kshiftlw $6, %k1, %k14254; AVX512F-NEXT: korw %k1, %k2, %k14255; AVX512F-NEXT: kshiftlw $9, %k1, %k14256; AVX512F-NEXT: kshiftrw $9, %k1, %k14257; AVX512F-NEXT: kshiftlw $7, %k0, %k04258; AVX512F-NEXT: korw %k0, %k1, %k04259; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload4260; AVX512F-NEXT: kshiftlw $9, %k1, %k14261; AVX512F-NEXT: kshiftrw $9, %k1, %k14262; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload4263; AVX512F-NEXT: kshiftlw $7, %k2, %k24264; AVX512F-NEXT: korw %k2, %k1, %k14265; AVX512F-NEXT: kxorw %k0, %k1, %k04266; AVX512F-NEXT: kshiftrw $4, %k0, %k14267; AVX512F-NEXT: kxorw %k1, %k0, %k04268; AVX512F-NEXT: kshiftrw $2, %k0, %k14269; AVX512F-NEXT: kxorw %k1, %k0, %k04270; AVX512F-NEXT: kshiftrw $1, %k0, %k14271; AVX512F-NEXT: kxorw %k1, %k0, %k04272; AVX512F-NEXT: kmovw %k0, %eax4273; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload4274; AVX512F-NEXT: vpcompressd %zmm2, %zmm2 {%k1} {z}4275; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload4276; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload4277; AVX512F-NEXT: kandw %k1, %k0, %k04278; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload4279; AVX512F-NEXT: kshiftrw $14, %k1, %k14280; AVX512F-NEXT: korw %k1, %k0, %k04281; AVX512F-NEXT: kandw %k7, %k0, %k04282; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload4283; AVX512F-NEXT: kshiftrw $13, %k1, %k14284; AVX512F-NEXT: korw %k1, %k0, %k04285; AVX512F-NEXT: kandw %k6, %k0, %k04286; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload4287; AVX512F-NEXT: kshiftrw $12, %k1, %k14288; AVX512F-NEXT: korw %k1, %k0, %k04289; AVX512F-NEXT: kandw %k5, %k0, %k04290; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload4291; AVX512F-NEXT: kshiftrw $11, %k1, %k14292; AVX512F-NEXT: korw %k1, %k0, %k04293; AVX512F-NEXT: kandw %k4, %k0, %k04294; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload4295; AVX512F-NEXT: kshiftrw $10, %k1, %k14296; AVX512F-NEXT: korw %k1, %k0, %k04297; AVX512F-NEXT: kandw %k3, %k0, %k04298; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload4299; AVX512F-NEXT: kshiftlw $6, %k1, %k14300; AVX512F-NEXT: korw %k1, %k0, %k04301; AVX512F-NEXT: kshiftlw $9, %k0, %k04302; AVX512F-NEXT: kshiftrw $9, %k0, %k04303; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload4304; AVX512F-NEXT: kshiftlw $7, %k1, %k14305; AVX512F-NEXT: korw %k1, %k0, %k04306; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload4307; AVX512F-NEXT: kshiftlw $9, %k1, %k14308; AVX512F-NEXT: kshiftrw $9, %k1, %k14309; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload4310; AVX512F-NEXT: kshiftlw $7, %k2, %k24311; AVX512F-NEXT: korw %k2, %k1, %k14312; AVX512F-NEXT: kxorw %k0, %k1, %k04313; AVX512F-NEXT: kshiftrw $4, %k0, %k14314; AVX512F-NEXT: kxorw %k1, %k0, %k04315; AVX512F-NEXT: kshiftrw $2, %k0, %k14316; AVX512F-NEXT: kxorw %k1, %k0, %k04317; AVX512F-NEXT: kshiftrw $1, %k0, %k14318; AVX512F-NEXT: kxorw %k1, %k0, %k04319; AVX512F-NEXT: kmovw %k0, %ecx4320; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload4321; AVX512F-NEXT: vpcompressd %zmm3, %zmm3 {%k1} {z}4322; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload4323; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k2} {z}4324; AVX512F-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload4325; AVX512F-NEXT: vpcompressd %zmm1, %zmm1 {%k1} {z}4326; AVX512F-NEXT: kxorw %k1, %k2, %k04327; AVX512F-NEXT: kshiftrw $8, %k0, %k14328; AVX512F-NEXT: kxorw %k1, %k0, %k04329; AVX512F-NEXT: kshiftrw $4, %k0, %k14330; AVX512F-NEXT: kxorw %k1, %k0, %k04331; AVX512F-NEXT: kshiftrw $2, %k0, %k14332; AVX512F-NEXT: kxorw %k1, %k0, %k04333; AVX512F-NEXT: kshiftrw $1, %k0, %k14334; AVX512F-NEXT: kxorw %k1, %k0, %k04335; AVX512F-NEXT: kmovw %k0, %edx4336; AVX512F-NEXT: vmovdqa64 %zmm0, {{[0-9]+}}(%rsp)4337; AVX512F-NEXT: andl $31, %eax4338; AVX512F-NEXT: vmovdqa64 %zmm1, 64(%rsp,%rax,4)4339; AVX512F-NEXT: vmovdqa64 %zmm2, {{[0-9]+}}(%rsp)4340; AVX512F-NEXT: andl $31, %ecx4341; AVX512F-NEXT: vmovdqa64 %zmm3, 192(%rsp,%rcx,4)4342; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm04343; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm14344; AVX512F-NEXT: vmovaps %zmm0, {{[0-9]+}}(%rsp)4345; AVX512F-NEXT: andl $63, %edx4346; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm04347; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm24348; AVX512F-NEXT: vmovaps %zmm0, 320(%rsp,%rdx,4)4349; AVX512F-NEXT: vmovaps %zmm1, {{[0-9]+}}(%rsp)4350; AVX512F-NEXT: vmovaps %zmm2, 384(%rsp,%rdx,4)4351; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm04352; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm14353; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm24354; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm34355; AVX512F-NEXT: movq %rbp, %rsp4356; AVX512F-NEXT: popq %rbp4357; AVX512F-NEXT: retq4358;4359; AVX512VL-LABEL: test_compress_large:4360; AVX512VL: # %bb.0:4361; AVX512VL-NEXT: pushq %rbp4362; AVX512VL-NEXT: movq %rsp, %rbp4363; AVX512VL-NEXT: andq $-64, %rsp4364; AVX512VL-NEXT: subq $576, %rsp # imm = 0x2404365; AVX512VL-NEXT: vpsllw $7, %zmm0, %zmm04366; AVX512VL-NEXT: vpmovb2m %zmm0, %k14367; AVX512VL-NEXT: kshiftrq $48, %k1, %k34368; AVX512VL-NEXT: kshiftrq $32, %k1, %k44369; AVX512VL-NEXT: kshiftrq $16, %k1, %k24370; AVX512VL-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}4371; AVX512VL-NEXT: vmovdqa64 %zmm0, (%rsp)4372; AVX512VL-NEXT: kshiftrq $8, %k1, %k04373; AVX512VL-NEXT: kxorw %k0, %k1, %k04374; AVX512VL-NEXT: kshiftrw $4, %k0, %k54375; AVX512VL-NEXT: kxorw %k5, %k0, %k04376; AVX512VL-NEXT: kshiftrw $2, %k0, %k54377; AVX512VL-NEXT: kxorw %k5, %k0, %k04378; AVX512VL-NEXT: kshiftrw $1, %k0, %k54379; AVX512VL-NEXT: kxorw %k5, %k0, %k04380; AVX512VL-NEXT: kmovd %k0, %eax4381; AVX512VL-NEXT: andl $31, %eax4382; AVX512VL-NEXT: vpcompressd %zmm2, %zmm0 {%k2} {z}4383; AVX512VL-NEXT: vmovdqa64 %zmm0, (%rsp,%rax,4)4384; AVX512VL-NEXT: vpcompressd %zmm3, %zmm0 {%k4} {z}4385; AVX512VL-NEXT: vmovdqa64 %zmm0, {{[0-9]+}}(%rsp)4386; AVX512VL-NEXT: kshiftrq $40, %k1, %k04387; AVX512VL-NEXT: kxorw %k0, %k4, %k04388; AVX512VL-NEXT: kshiftrw $4, %k0, %k44389; AVX512VL-NEXT: kxorw %k4, %k0, %k04390; AVX512VL-NEXT: kshiftrw $2, %k0, %k44391; AVX512VL-NEXT: kxorw %k4, %k0, %k04392; AVX512VL-NEXT: kshiftrw $1, %k0, %k44393; AVX512VL-NEXT: kxorw %k4, %k0, %k04394; AVX512VL-NEXT: kmovd %k0, %eax4395; AVX512VL-NEXT: andl $31, %eax4396; AVX512VL-NEXT: vpcompressd %zmm4, %zmm0 {%k3} {z}4397; AVX512VL-NEXT: vmovdqa64 %zmm0, 128(%rsp,%rax,4)4398; AVX512VL-NEXT: vmovaps (%rsp), %zmm04399; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm14400; AVX512VL-NEXT: vmovaps %zmm0, {{[0-9]+}}(%rsp)4401; AVX512VL-NEXT: kxorw %k2, %k1, %k04402; AVX512VL-NEXT: kshiftrw $8, %k0, %k14403; AVX512VL-NEXT: kxorw %k1, %k0, %k04404; AVX512VL-NEXT: kshiftrw $4, %k0, %k14405; AVX512VL-NEXT: kxorw %k1, %k0, %k04406; AVX512VL-NEXT: kshiftrw $2, %k0, %k14407; AVX512VL-NEXT: kxorw %k1, %k0, %k04408; AVX512VL-NEXT: kshiftrw $1, %k0, %k14409; AVX512VL-NEXT: kxorw %k1, %k0, %k04410; AVX512VL-NEXT: kmovd %k0, %eax4411; AVX512VL-NEXT: andl $63, %eax4412; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm04413; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm24414; AVX512VL-NEXT: vmovaps %zmm0, 256(%rsp,%rax,4)4415; AVX512VL-NEXT: vmovaps %zmm1, {{[0-9]+}}(%rsp)4416; AVX512VL-NEXT: vmovaps %zmm2, 320(%rsp,%rax,4)4417; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm04418; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm14419; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm24420; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm34421; AVX512VL-NEXT: movq %rbp, %rsp4422; AVX512VL-NEXT: popq %rbp4423; AVX512VL-NEXT: retq4424 %out = call <64 x i32> @llvm.experimental.vector.compress(<64 x i32> %vec, <64 x i1> %mask, <64 x i32> undef)4425 ret <64 x i32> %out4426}4427 4428define <8 x i64> @test_compress_knownbits_zext_v8i16_8i64(<8 x i16> %vec, <8 x i1> %mask, <8 x i64> %passthru) nounwind {4429; AVX2-LABEL: test_compress_knownbits_zext_v8i16_8i64:4430; AVX2: # %bb.0:4431; AVX2-NEXT: pushq %rbp4432; AVX2-NEXT: movq %rsp, %rbp4433; AVX2-NEXT: pushq %rbx4434; AVX2-NEXT: andq $-32, %rsp4435; AVX2-NEXT: subq $96, %rsp4436; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero4437; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]4438; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero4439; AVX2-NEXT: vbroadcastsd {{.*#+}} ymm5 = [3,3,3,3]4440; AVX2-NEXT: vandps %ymm5, %ymm2, %ymm24441; AVX2-NEXT: vandps %ymm5, %ymm3, %ymm34442; AVX2-NEXT: vmovaps %ymm3, {{[0-9]+}}(%rsp)4443; AVX2-NEXT: vmovaps %ymm2, (%rsp)4444; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm24445; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero4446; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]4447; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero4448; AVX2-NEXT: vpaddq %ymm2, %ymm3, %ymm24449; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm34450; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm24451; AVX2-NEXT: vpextrq $1, %xmm2, %rcx4452; AVX2-NEXT: vmovq %xmm2, %rax4453; AVX2-NEXT: addl %ecx, %eax4454; AVX2-NEXT: andl $7, %eax4455; AVX2-NEXT: vpextrw $1, %xmm1, %ecx4456; AVX2-NEXT: andl $1, %ecx4457; AVX2-NEXT: vmovd %xmm1, %edx4458; AVX2-NEXT: andl $1, %edx4459; AVX2-NEXT: addq %rdx, %rcx4460; AVX2-NEXT: vpextrw $2, %xmm1, %esi4461; AVX2-NEXT: andl $1, %esi4462; AVX2-NEXT: addq %rcx, %rsi4463; AVX2-NEXT: vpextrw $3, %xmm1, %edi4464; AVX2-NEXT: andl $1, %edi4465; AVX2-NEXT: addq %rsi, %rdi4466; AVX2-NEXT: vpextrw $4, %xmm1, %r8d4467; AVX2-NEXT: andl $1, %r8d4468; AVX2-NEXT: addq %rdi, %r84469; AVX2-NEXT: vpextrw $5, %xmm1, %r9d4470; AVX2-NEXT: andl $1, %r9d4471; AVX2-NEXT: addq %r8, %r94472; AVX2-NEXT: vpextrw $6, %xmm1, %r10d4473; AVX2-NEXT: andl $1, %r10d4474; AVX2-NEXT: addq %r9, %r104475; AVX2-NEXT: vpextrw $7, %xmm1, %r11d4476; AVX2-NEXT: andl $1, %r11d4477; AVX2-NEXT: addq %r10, %r114478; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm14479; AVX2-NEXT: vpextrq $1, %xmm1, %rbx4480; AVX2-NEXT: cmpq $8, %r114481; AVX2-NEXT: cmovbq (%rsp,%rax,8), %rbx4482; AVX2-NEXT: vmovq %xmm4, (%rsp)4483; AVX2-NEXT: vpextrq $1, %xmm4, (%rsp,%rdx,8)4484; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm24485; AVX2-NEXT: vmovq %xmm2, (%rsp,%rcx,8)4486; AVX2-NEXT: vpextrq $1, %xmm2, (%rsp,%rsi,8)4487; AVX2-NEXT: andl $7, %edi4488; AVX2-NEXT: vmovq %xmm0, (%rsp,%rdi,8)4489; AVX2-NEXT: andl $7, %r8d4490; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%r8,8)4491; AVX2-NEXT: andl $7, %r9d4492; AVX2-NEXT: vmovq %xmm1, (%rsp,%r9,8)4493; AVX2-NEXT: andl $7, %r10d4494; AVX2-NEXT: vpextrq $1, %xmm1, (%rsp,%r10,8)4495; AVX2-NEXT: cmpq $7, %r114496; AVX2-NEXT: movl $7, %eax4497; AVX2-NEXT: cmovbq %r11, %rax4498; AVX2-NEXT: movl %eax, %eax4499; AVX2-NEXT: movq %rbx, (%rsp,%rax,8)4500; AVX2-NEXT: vmovaps (%rsp), %ymm04501; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm14502; AVX2-NEXT: leaq -8(%rbp), %rsp4503; AVX2-NEXT: popq %rbx4504; AVX2-NEXT: popq %rbp4505; AVX2-NEXT: retq4506;4507; AVX512F-LABEL: test_compress_knownbits_zext_v8i16_8i64:4508; AVX512F: # %bb.0:4509; AVX512F-NEXT: vpmovsxwq %xmm1, %zmm14510; AVX512F-NEXT: vpsllq $63, %zmm1, %zmm14511; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k14512; AVX512F-NEXT: vpmovzxwq {{.*#+}} zmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero4513; AVX512F-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm04514; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1}4515; AVX512F-NEXT: retq4516;4517; AVX512VL-LABEL: test_compress_knownbits_zext_v8i16_8i64:4518; AVX512VL: # %bb.0:4519; AVX512VL-NEXT: vpsllw $15, %xmm1, %xmm14520; AVX512VL-NEXT: vpmovw2m %xmm1, %k14521; AVX512VL-NEXT: vpmovzxwq {{.*#+}} zmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero4522; AVX512VL-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm04523; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1}4524; AVX512VL-NEXT: retq4525 %xvec = zext <8 x i16> %vec to <8 x i64> ; 0 -> 655354526 %xpassthru = and <8 x i64> %passthru, splat (i64 3) ; 0 -> 34527 %out = call <8 x i64> @llvm.experimental.vector.compress(<8 x i64> %xvec, <8 x i1> %mask, <8 x i64> %xpassthru)4528 %res = and <8 x i64> %out, splat (i64 65535) ; unnecessary - %out guaranteed to be 0 -> 655354529 ret <8 x i64> %res4530}4531 4532define <8 x i64> @test_compress_knownbits_sext_v8i16_8i64(<8 x i16> %vec, <8 x i1> %mask, <8 x i64> %passthru) nounwind {4533; AVX2-LABEL: test_compress_knownbits_sext_v8i16_8i64:4534; AVX2: # %bb.0:4535; AVX2-NEXT: pushq %rbp4536; AVX2-NEXT: movq %rsp, %rbp4537; AVX2-NEXT: pushq %rbx4538; AVX2-NEXT: andq $-32, %rsp4539; AVX2-NEXT: subq $96, %rsp4540; AVX2-NEXT: vpmovsxwq %xmm0, %ymm44541; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]4542; AVX2-NEXT: vpmovsxwq %xmm0, %ymm04543; AVX2-NEXT: vbroadcastsd {{.*#+}} ymm5 = [3,3,3,3]4544; AVX2-NEXT: vandps %ymm5, %ymm2, %ymm24545; AVX2-NEXT: vandps %ymm5, %ymm3, %ymm34546; AVX2-NEXT: vmovaps %ymm3, {{[0-9]+}}(%rsp)4547; AVX2-NEXT: vmovaps %ymm2, (%rsp)4548; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm24549; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero4550; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]4551; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero4552; AVX2-NEXT: vpaddq %ymm2, %ymm3, %ymm24553; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm34554; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm24555; AVX2-NEXT: vpextrq $1, %xmm2, %rcx4556; AVX2-NEXT: vmovq %xmm2, %rax4557; AVX2-NEXT: addl %ecx, %eax4558; AVX2-NEXT: andl $7, %eax4559; AVX2-NEXT: vpextrw $1, %xmm1, %ecx4560; AVX2-NEXT: andl $1, %ecx4561; AVX2-NEXT: vmovd %xmm1, %edx4562; AVX2-NEXT: andl $1, %edx4563; AVX2-NEXT: addq %rdx, %rcx4564; AVX2-NEXT: vpextrw $2, %xmm1, %esi4565; AVX2-NEXT: andl $1, %esi4566; AVX2-NEXT: addq %rcx, %rsi4567; AVX2-NEXT: vpextrw $3, %xmm1, %edi4568; AVX2-NEXT: andl $1, %edi4569; AVX2-NEXT: addq %rsi, %rdi4570; AVX2-NEXT: vpextrw $4, %xmm1, %r8d4571; AVX2-NEXT: andl $1, %r8d4572; AVX2-NEXT: addq %rdi, %r84573; AVX2-NEXT: vpextrw $5, %xmm1, %r9d4574; AVX2-NEXT: andl $1, %r9d4575; AVX2-NEXT: addq %r8, %r94576; AVX2-NEXT: vpextrw $6, %xmm1, %r10d4577; AVX2-NEXT: andl $1, %r10d4578; AVX2-NEXT: addq %r9, %r104579; AVX2-NEXT: vpextrw $7, %xmm1, %r11d4580; AVX2-NEXT: andl $1, %r11d4581; AVX2-NEXT: addq %r10, %r114582; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm14583; AVX2-NEXT: vpextrq $1, %xmm1, %rbx4584; AVX2-NEXT: cmpq $8, %r114585; AVX2-NEXT: cmovbq (%rsp,%rax,8), %rbx4586; AVX2-NEXT: vmovq %xmm4, (%rsp)4587; AVX2-NEXT: vpextrq $1, %xmm4, (%rsp,%rdx,8)4588; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm24589; AVX2-NEXT: vmovq %xmm2, (%rsp,%rcx,8)4590; AVX2-NEXT: vpextrq $1, %xmm2, (%rsp,%rsi,8)4591; AVX2-NEXT: andl $7, %edi4592; AVX2-NEXT: vmovq %xmm0, (%rsp,%rdi,8)4593; AVX2-NEXT: andl $7, %r8d4594; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%r8,8)4595; AVX2-NEXT: andl $7, %r9d4596; AVX2-NEXT: vmovq %xmm1, (%rsp,%r9,8)4597; AVX2-NEXT: andl $7, %r10d4598; AVX2-NEXT: vpextrq $1, %xmm1, (%rsp,%r10,8)4599; AVX2-NEXT: cmpq $7, %r114600; AVX2-NEXT: movl $7, %eax4601; AVX2-NEXT: cmovbq %r11, %rax4602; AVX2-NEXT: movl %eax, %eax4603; AVX2-NEXT: movq %rbx, (%rsp,%rax,8)4604; AVX2-NEXT: vmovaps (%rsp), %ymm04605; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %ymm14606; AVX2-NEXT: leaq -8(%rbp), %rsp4607; AVX2-NEXT: popq %rbx4608; AVX2-NEXT: popq %rbp4609; AVX2-NEXT: retq4610;4611; AVX512F-LABEL: test_compress_knownbits_sext_v8i16_8i64:4612; AVX512F: # %bb.0:4613; AVX512F-NEXT: vpmovsxwq %xmm1, %zmm14614; AVX512F-NEXT: vpsllq $63, %zmm1, %zmm14615; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k14616; AVX512F-NEXT: vpmovsxwq %xmm0, %zmm14617; AVX512F-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm04618; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1}4619; AVX512F-NEXT: retq4620;4621; AVX512VL-LABEL: test_compress_knownbits_sext_v8i16_8i64:4622; AVX512VL: # %bb.0:4623; AVX512VL-NEXT: vpsllw $15, %xmm1, %xmm14624; AVX512VL-NEXT: vpmovw2m %xmm1, %k14625; AVX512VL-NEXT: vpmovsxwq %xmm0, %zmm14626; AVX512VL-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm04627; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1}4628; AVX512VL-NEXT: retq4629 %xvec = sext <8 x i16> %vec to <8 x i64> ; sign extend vec4630 %xpassthru = and <8 x i64> %passthru, splat(i64 3)4631 %out = call <8 x i64> @llvm.experimental.vector.compress(<8 x i64> %xvec, <8 x i1> %mask, <8 x i64> %xpassthru)4632 %shl = shl <8 x i64> %out, splat(i64 48)4633 %res = ashr <8 x i64> %shl, splat(i64 48)4634 ret <8 x i64> %res4635}4636 4637define <4 x i32> @test_compress_all_const() nounwind {4638; AVX2-LABEL: test_compress_all_const:4639; AVX2: # %bb.0:4640; AVX2-NEXT: vmovsd {{.*#+}} xmm0 = [5,9,0,0]4641; AVX2-NEXT: retq4642;4643; AVX512F-LABEL: test_compress_all_const:4644; AVX512F: # %bb.0:4645; AVX512F-NEXT: vpmovsxbd {{.*#+}} xmm0 = [5,9,0,0]4646; AVX512F-NEXT: retq4647;4648; AVX512VL-LABEL: test_compress_all_const:4649; AVX512VL: # %bb.0:4650; AVX512VL-NEXT: vpmovsxbd {{.*#+}} xmm0 = [5,9,0,0]4651; AVX512VL-NEXT: retq4652 %out = call <4 x i32> @llvm.experimental.vector.compress(<4 x i32> <i32 3, i32 5, i32 7, i32 9>,4653 <4 x i1> <i1 0, i1 1, i1 0, i1 1>,4654 <4 x i32> undef)4655 ret <4 x i32> %out4656}4657 4658define <4 x i32> @test_compress_const_mask(<4 x i32> %vec) nounwind {4659; CHECK-LABEL: test_compress_const_mask:4660; CHECK: # %bb.0:4661; CHECK-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,3,2,3]4662; CHECK-NEXT: retq4663 %out = call <4 x i32> @llvm.experimental.vector.compress(<4 x i32> %vec, <4 x i1> <i1 1, i1 undef, i1 0, i1 1>, <4 x i32> undef)4664 ret <4 x i32> %out4665}4666 4667define <4 x i32> @test_compress_const_mask_passthrough(<4 x i32> %vec, <4 x i32> %passthru) nounwind {4668; CHECK-LABEL: test_compress_const_mask_passthrough:4669; CHECK: # %bb.0:4670; CHECK-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,3],xmm1[2,3]4671; CHECK-NEXT: retq4672 %out = call <4 x i32> @llvm.experimental.vector.compress(<4 x i32> %vec, <4 x i1> <i1 1, i1 undef, i1 0, i1 1>, <4 x i32> %passthru)4673 ret <4 x i32> %out4674}4675 4676define <4 x i32> @test_compress_const_mask_const_passthrough(<4 x i32> %vec) nounwind {4677; CHECK-LABEL: test_compress_const_mask_const_passthrough:4678; CHECK: # %bb.0:4679; CHECK-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,3,2,3]4680; CHECK-NEXT: movl $7, %eax4681; CHECK-NEXT: vpinsrd $2, %eax, %xmm0, %xmm04682; CHECK-NEXT: movl $8, %eax4683; CHECK-NEXT: vpinsrd $3, %eax, %xmm0, %xmm04684; CHECK-NEXT: retq4685 %out = call <4 x i32> @llvm.experimental.vector.compress(<4 x i32> %vec, <4 x i1> <i1 1, i1 0, i1 0, i1 1>, <4 x i32> <i32 5, i32 6, i32 7, i32 8>)4686 ret <4 x i32> %out4687}4688 4689; We pass a placeholder value for the const_mask* tests to check that they are converted to a no-op by simply copying4690; the second vector input register to the return register or doing nothing.4691define <4 x i32> @test_compress_const_splat1_mask(<4 x i32> %ignore, <4 x i32> %vec) nounwind {4692; CHECK-LABEL: test_compress_const_splat1_mask:4693; CHECK: # %bb.0:4694; CHECK-NEXT: vmovaps %xmm1, %xmm04695; CHECK-NEXT: retq4696 %out = call <4 x i32> @llvm.experimental.vector.compress(<4 x i32> %vec, <4 x i1> splat (i1 -1), <4 x i32> undef)4697 ret <4 x i32> %out4698}4699define <4 x i32> @test_compress_const_splat0_mask(<4 x i32> %ignore, <4 x i32> %vec) nounwind {4700; CHECK-LABEL: test_compress_const_splat0_mask:4701; CHECK: # %bb.0:4702; CHECK-NEXT: retq4703 %out = call <4 x i32> @llvm.experimental.vector.compress(<4 x i32> %vec, <4 x i1> splat (i1 0), <4 x i32> undef)4704 ret <4 x i32> %out4705}4706define <4 x i32> @test_compress_undef_mask(<4 x i32> %ignore, <4 x i32> %vec) nounwind {4707; CHECK-LABEL: test_compress_undef_mask:4708; CHECK: # %bb.0:4709; CHECK-NEXT: retq4710 %out = call <4 x i32> @llvm.experimental.vector.compress(<4 x i32> %vec, <4 x i1> undef, <4 x i32> undef)4711 ret <4 x i32> %out4712}4713define <4 x i32> @test_compress_const_splat0_mask_with_passthru(<4 x i32> %ignore, <4 x i32> %vec, <4 x i32> %passthru) nounwind {4714; CHECK-LABEL: test_compress_const_splat0_mask_with_passthru:4715; CHECK: # %bb.0:4716; CHECK-NEXT: vmovaps %xmm2, %xmm04717; CHECK-NEXT: retq4718 %out = call <4 x i32> @llvm.experimental.vector.compress(<4 x i32> %vec, <4 x i1> splat (i1 0), <4 x i32> %passthru)4719 ret <4 x i32> %out4720}4721define <4 x i32> @test_compress_const_splat0_mask_without_passthru(<4 x i32> %ignore, <4 x i32> %vec) nounwind {4722; CHECK-LABEL: test_compress_const_splat0_mask_without_passthru:4723; CHECK: # %bb.0:4724; CHECK-NEXT: retq4725 %out = call <4 x i32> @llvm.experimental.vector.compress(<4 x i32> %vec, <4 x i1> splat (i1 0), <4 x i32> undef)4726 ret <4 x i32> %out4727}4728 4729define <4 x i8> @test_compress_small(<4 x i8> %vec, <4 x i1> %mask) nounwind {4730; AVX2-LABEL: test_compress_small:4731; AVX2: # %bb.0:4732; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,4,8,12],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero4733; AVX2-NEXT: vpsllw $7, %xmm1, %xmm14734; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm24735; AVX2-NEXT: vpcmpgtb %xmm1, %xmm2, %xmm14736; AVX2-NEXT: vpextrb $0, %xmm0, -{{[0-9]+}}(%rsp)4737; AVX2-NEXT: vmovd %xmm1, %eax4738; AVX2-NEXT: andl $1, %eax4739; AVX2-NEXT: vpextrb $1, %xmm0, -24(%rsp,%rax)4740; AVX2-NEXT: vpextrb $1, %xmm1, %ecx4741; AVX2-NEXT: andl $1, %ecx4742; AVX2-NEXT: addq %rax, %rcx4743; AVX2-NEXT: vpextrb $2, %xmm0, -24(%rsp,%rcx)4744; AVX2-NEXT: vpextrb $2, %xmm1, %eax4745; AVX2-NEXT: andl $1, %eax4746; AVX2-NEXT: addq %rcx, %rax4747; AVX2-NEXT: vpextrb $3, %xmm0, -24(%rsp,%rax)4748; AVX2-NEXT: vpextrb $3, %xmm1, %ecx4749; AVX2-NEXT: andl $1, %ecx4750; AVX2-NEXT: addq %rax, %rcx4751; AVX2-NEXT: vpextrb $15, %xmm0, -24(%rsp,%rcx)4752; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm04753; AVX2-NEXT: retq4754;4755; AVX512F-LABEL: test_compress_small:4756; AVX512F: # %bb.0:4757; AVX512F-NEXT: vpslld $31, %xmm1, %xmm14758; AVX512F-NEXT: vptestmd %zmm1, %zmm1, %k04759; AVX512F-NEXT: kshiftlw $12, %k0, %k04760; AVX512F-NEXT: kshiftrw $12, %k0, %k14761; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero4762; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}4763; AVX512F-NEXT: vpmovdb %zmm0, %xmm04764; AVX512F-NEXT: vzeroupper4765; AVX512F-NEXT: retq4766;4767; AVX512VL-LABEL: test_compress_small:4768; AVX512VL: # %bb.0:4769; AVX512VL-NEXT: vpslld $31, %xmm1, %xmm14770; AVX512VL-NEXT: vptestmd %xmm1, %xmm1, %k14771; AVX512VL-NEXT: vpcompressb %xmm0, %xmm0 {%k1} {z}4772; AVX512VL-NEXT: retq4773 %out = call <4 x i8> @llvm.experimental.vector.compress(<4 x i8> %vec, <4 x i1> %mask, <4 x i8> undef)4774 ret <4 x i8> %out4775}4776 4777define <4 x i4> @test_compress_illegal_element_type(<4 x i4> %vec, <4 x i1> %mask) nounwind {4778; AVX2-LABEL: test_compress_illegal_element_type:4779; AVX2: # %bb.0:4780; AVX2-NEXT: vpslld $31, %xmm1, %xmm14781; AVX2-NEXT: vpsrad $31, %xmm1, %xmm14782; AVX2-NEXT: vmovss %xmm0, -{{[0-9]+}}(%rsp)4783; AVX2-NEXT: vmovd %xmm1, %eax4784; AVX2-NEXT: andl $1, %eax4785; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rax,4)4786; AVX2-NEXT: vpextrd $1, %xmm1, %ecx4787; AVX2-NEXT: subl %ecx, %eax4788; AVX2-NEXT: leal (,%rax,4), %ecx4789; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rcx)4790; AVX2-NEXT: vpextrd $2, %xmm1, %ecx4791; AVX2-NEXT: subl %ecx, %eax4792; AVX2-NEXT: andl $3, %eax4793; AVX2-NEXT: vextractps $3, %xmm0, -24(%rsp,%rax,4)4794; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm04795; AVX2-NEXT: retq4796;4797; AVX512F-LABEL: test_compress_illegal_element_type:4798; AVX512F: # %bb.0:4799; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm04800; AVX512F-NEXT: vpslld $31, %xmm1, %xmm14801; AVX512F-NEXT: vptestmd %zmm1, %zmm1, %k04802; AVX512F-NEXT: kshiftlw $12, %k0, %k04803; AVX512F-NEXT: kshiftrw $12, %k0, %k14804; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}4805; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm04806; AVX512F-NEXT: vzeroupper4807; AVX512F-NEXT: retq4808;4809; AVX512VL-LABEL: test_compress_illegal_element_type:4810; AVX512VL: # %bb.0:4811; AVX512VL-NEXT: vpslld $31, %xmm1, %xmm14812; AVX512VL-NEXT: vptestmd %xmm1, %xmm1, %k14813; AVX512VL-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}4814; AVX512VL-NEXT: retq4815 %out = call <4 x i4> @llvm.experimental.vector.compress(<4 x i4> %vec, <4 x i1> %mask, <4 x i4> undef)4816 ret <4 x i4> %out4817}4818 4819define <3 x i32> @test_compress_narrow(<3 x i32> %vec, <3 x i1> %mask) nounwind {4820; AVX2-LABEL: test_compress_narrow:4821; AVX2: # %bb.0:4822; AVX2-NEXT: vmovd %edi, %xmm14823; AVX2-NEXT: vpinsrd $1, %esi, %xmm1, %xmm14824; AVX2-NEXT: vpinsrd $2, %edx, %xmm1, %xmm14825; AVX2-NEXT: vpslld $31, %xmm1, %xmm14826; AVX2-NEXT: vpsrad $31, %xmm1, %xmm14827; AVX2-NEXT: vmovss %xmm0, -{{[0-9]+}}(%rsp)4828; AVX2-NEXT: vmovd %xmm1, %eax4829; AVX2-NEXT: andl $1, %eax4830; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rax,4)4831; AVX2-NEXT: vpextrd $1, %xmm1, %ecx4832; AVX2-NEXT: subl %ecx, %eax4833; AVX2-NEXT: leal (,%rax,4), %ecx4834; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rcx)4835; AVX2-NEXT: vpextrd $2, %xmm1, %ecx4836; AVX2-NEXT: subl %ecx, %eax4837; AVX2-NEXT: andl $3, %eax4838; AVX2-NEXT: vextractps $3, %xmm0, -24(%rsp,%rax,4)4839; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm04840; AVX2-NEXT: retq4841;4842; AVX512F-LABEL: test_compress_narrow:4843; AVX512F: # %bb.0:4844; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm04845; AVX512F-NEXT: andl $1, %edi4846; AVX512F-NEXT: kmovw %edi, %k04847; AVX512F-NEXT: kmovw %esi, %k14848; AVX512F-NEXT: kshiftlw $15, %k1, %k14849; AVX512F-NEXT: kshiftrw $14, %k1, %k14850; AVX512F-NEXT: korw %k1, %k0, %k04851; AVX512F-NEXT: movw $-5, %ax4852; AVX512F-NEXT: kmovw %eax, %k14853; AVX512F-NEXT: kandw %k1, %k0, %k04854; AVX512F-NEXT: kmovw %edx, %k14855; AVX512F-NEXT: kshiftlw $15, %k1, %k14856; AVX512F-NEXT: kshiftrw $13, %k1, %k14857; AVX512F-NEXT: korw %k1, %k0, %k04858; AVX512F-NEXT: movb $7, %al4859; AVX512F-NEXT: kmovw %eax, %k14860; AVX512F-NEXT: kandw %k1, %k0, %k04861; AVX512F-NEXT: kshiftlw $12, %k0, %k04862; AVX512F-NEXT: kshiftrw $12, %k0, %k14863; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}4864; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm04865; AVX512F-NEXT: vzeroupper4866; AVX512F-NEXT: retq4867;4868; AVX512VL-LABEL: test_compress_narrow:4869; AVX512VL: # %bb.0:4870; AVX512VL-NEXT: andl $1, %edi4871; AVX512VL-NEXT: kmovw %edi, %k04872; AVX512VL-NEXT: kmovd %esi, %k14873; AVX512VL-NEXT: kshiftlw $15, %k1, %k14874; AVX512VL-NEXT: kshiftrw $14, %k1, %k14875; AVX512VL-NEXT: korw %k1, %k0, %k04876; AVX512VL-NEXT: movw $-5, %ax4877; AVX512VL-NEXT: kmovd %eax, %k14878; AVX512VL-NEXT: kandw %k1, %k0, %k04879; AVX512VL-NEXT: kmovd %edx, %k14880; AVX512VL-NEXT: kshiftlw $15, %k1, %k14881; AVX512VL-NEXT: kshiftrw $13, %k1, %k14882; AVX512VL-NEXT: korw %k1, %k0, %k04883; AVX512VL-NEXT: movb $7, %al4884; AVX512VL-NEXT: kmovd %eax, %k14885; AVX512VL-NEXT: kandw %k1, %k0, %k14886; AVX512VL-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}4887; AVX512VL-NEXT: retq4888 %out = call <3 x i32> @llvm.experimental.vector.compress(<3 x i32> %vec, <3 x i1> %mask, <3 x i32> undef)4889 ret <3 x i32> %out4890}4891 4892define <3 x i3> @test_compress_narrow_illegal_element_type(<3 x i3> %vec, <3 x i1> %mask) nounwind {4893; AVX2-LABEL: test_compress_narrow_illegal_element_type:4894; AVX2: # %bb.0:4895; AVX2-NEXT: vmovd %ecx, %xmm04896; AVX2-NEXT: vpinsrd $1, %r8d, %xmm0, %xmm04897; AVX2-NEXT: vpslld $31, %xmm0, %xmm04898; AVX2-NEXT: vpsrad $31, %xmm0, %xmm04899; AVX2-NEXT: movl %edi, -{{[0-9]+}}(%rsp)4900; AVX2-NEXT: vmovd %xmm0, %eax4901; AVX2-NEXT: andl $1, %eax4902; AVX2-NEXT: movl %esi, -24(%rsp,%rax,4)4903; AVX2-NEXT: vpextrd $1, %xmm0, %ecx4904; AVX2-NEXT: subl %ecx, %eax4905; AVX2-NEXT: shll $2, %eax4906; AVX2-NEXT: movl %edx, -24(%rsp,%rax)4907; AVX2-NEXT: vmovdqa -{{[0-9]+}}(%rsp), %xmm04908; AVX2-NEXT: vmovd %xmm0, %eax4909; AVX2-NEXT: vpextrb $4, %xmm0, %edx4910; AVX2-NEXT: vpextrb $8, %xmm0, %ecx4911; AVX2-NEXT: # kill: def $al killed $al killed $eax4912; AVX2-NEXT: # kill: def $dl killed $dl killed $edx4913; AVX2-NEXT: # kill: def $cl killed $cl killed $ecx4914; AVX2-NEXT: retq4915;4916; AVX512F-LABEL: test_compress_narrow_illegal_element_type:4917; AVX512F: # %bb.0:4918; AVX512F-NEXT: andl $1, %ecx4919; AVX512F-NEXT: kmovw %ecx, %k04920; AVX512F-NEXT: kmovw %r8d, %k14921; AVX512F-NEXT: kshiftlw $15, %k1, %k14922; AVX512F-NEXT: kshiftrw $14, %k1, %k14923; AVX512F-NEXT: korw %k1, %k0, %k04924; AVX512F-NEXT: movw $-5, %ax4925; AVX512F-NEXT: kmovw %eax, %k14926; AVX512F-NEXT: kandw %k1, %k0, %k04927; AVX512F-NEXT: kmovw %r9d, %k14928; AVX512F-NEXT: kshiftlw $15, %k1, %k14929; AVX512F-NEXT: kshiftrw $13, %k1, %k14930; AVX512F-NEXT: korw %k1, %k0, %k04931; AVX512F-NEXT: movb $7, %al4932; AVX512F-NEXT: kmovw %eax, %k14933; AVX512F-NEXT: kandw %k1, %k0, %k04934; AVX512F-NEXT: vmovd %edi, %xmm04935; AVX512F-NEXT: vpinsrd $1, %esi, %xmm0, %xmm04936; AVX512F-NEXT: vpinsrd $2, %edx, %xmm0, %xmm04937; AVX512F-NEXT: kshiftlw $12, %k0, %k04938; AVX512F-NEXT: kshiftrw $12, %k0, %k14939; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}4940; AVX512F-NEXT: vmovd %xmm0, %eax4941; AVX512F-NEXT: vpextrb $4, %xmm0, %edx4942; AVX512F-NEXT: vpextrb $8, %xmm0, %ecx4943; AVX512F-NEXT: # kill: def $al killed $al killed $eax4944; AVX512F-NEXT: # kill: def $dl killed $dl killed $edx4945; AVX512F-NEXT: # kill: def $cl killed $cl killed $ecx4946; AVX512F-NEXT: vzeroupper4947; AVX512F-NEXT: retq4948;4949; AVX512VL-LABEL: test_compress_narrow_illegal_element_type:4950; AVX512VL: # %bb.0:4951; AVX512VL-NEXT: andl $1, %ecx4952; AVX512VL-NEXT: kmovw %ecx, %k04953; AVX512VL-NEXT: kmovd %r8d, %k14954; AVX512VL-NEXT: kshiftlw $15, %k1, %k14955; AVX512VL-NEXT: kshiftrw $14, %k1, %k14956; AVX512VL-NEXT: korw %k1, %k0, %k04957; AVX512VL-NEXT: movw $-5, %ax4958; AVX512VL-NEXT: kmovd %eax, %k14959; AVX512VL-NEXT: kandw %k1, %k0, %k04960; AVX512VL-NEXT: kmovd %r9d, %k14961; AVX512VL-NEXT: kshiftlw $15, %k1, %k14962; AVX512VL-NEXT: kshiftrw $13, %k1, %k14963; AVX512VL-NEXT: korw %k1, %k0, %k04964; AVX512VL-NEXT: movb $7, %al4965; AVX512VL-NEXT: kmovd %eax, %k14966; AVX512VL-NEXT: kandw %k1, %k0, %k14967; AVX512VL-NEXT: vmovd %edi, %xmm04968; AVX512VL-NEXT: vpinsrd $1, %esi, %xmm0, %xmm04969; AVX512VL-NEXT: vpinsrd $2, %edx, %xmm0, %xmm04970; AVX512VL-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}4971; AVX512VL-NEXT: vmovd %xmm0, %eax4972; AVX512VL-NEXT: vpextrb $4, %xmm0, %edx4973; AVX512VL-NEXT: vpextrb $8, %xmm0, %ecx4974; AVX512VL-NEXT: # kill: def $al killed $al killed $eax4975; AVX512VL-NEXT: # kill: def $dl killed $dl killed $edx4976; AVX512VL-NEXT: # kill: def $cl killed $cl killed $ecx4977; AVX512VL-NEXT: retq4978 %out = call <3 x i3> @llvm.experimental.vector.compress(<3 x i3> %vec, <3 x i1> %mask, <3 x i3> undef)4979 ret <3 x i3> %out4980}4981 4982define <4 x i32> @test_compress_v4i32_zero_passthru(<4 x i32> %vec, <4 x i1> %mask) nounwind {4983; AVX2-LABEL: test_compress_v4i32_zero_passthru:4984; AVX2: # %bb.0:4985; AVX2-NEXT: vpslld $31, %xmm1, %xmm14986; AVX2-NEXT: vpsrad $31, %xmm1, %xmm14987; AVX2-NEXT: vxorps %xmm2, %xmm2, %xmm24988; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)4989; AVX2-NEXT: vmovss %xmm0, -{{[0-9]+}}(%rsp)4990; AVX2-NEXT: vmovd %xmm1, %eax4991; AVX2-NEXT: andl $1, %eax4992; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rax,4)4993; AVX2-NEXT: vpextrd $1, %xmm1, %ecx4994; AVX2-NEXT: andl $1, %ecx4995; AVX2-NEXT: addq %rax, %rcx4996; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rcx,4)4997; AVX2-NEXT: vpextrd $2, %xmm1, %eax4998; AVX2-NEXT: andl $1, %eax4999; AVX2-NEXT: addq %rcx, %rax5000; AVX2-NEXT: vpextrd $3, %xmm1, %ecx5001; AVX2-NEXT: andl $1, %ecx5002; AVX2-NEXT: addq %rax, %rcx5003; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax5004; AVX2-NEXT: andl $3, %eax5005; AVX2-NEXT: vextractps $3, %xmm0, -24(%rsp,%rax,4)5006; AVX2-NEXT: xorl %eax, %eax5007; AVX2-NEXT: cmpq $3, %rcx5008; AVX2-NEXT: movl $3, %edx5009; AVX2-NEXT: cmovbq %rcx, %rdx5010; AVX2-NEXT: vextractps $3, %xmm0, %ecx5011; AVX2-NEXT: cmovbel %eax, %ecx5012; AVX2-NEXT: movl %ecx, -24(%rsp,%rdx,4)5013; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm05014; AVX2-NEXT: retq5015;5016; AVX512F-LABEL: test_compress_v4i32_zero_passthru:5017; AVX512F: # %bb.0:5018; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm05019; AVX512F-NEXT: vpslld $31, %xmm1, %xmm15020; AVX512F-NEXT: vptestmd %zmm1, %zmm1, %k05021; AVX512F-NEXT: kshiftlw $12, %k0, %k05022; AVX512F-NEXT: kshiftrw $12, %k0, %k15023; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}5024; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm05025; AVX512F-NEXT: vzeroupper5026; AVX512F-NEXT: retq5027;5028; AVX512VL-LABEL: test_compress_v4i32_zero_passthru:5029; AVX512VL: # %bb.0:5030; AVX512VL-NEXT: vpslld $31, %xmm1, %xmm15031; AVX512VL-NEXT: vptestmd %xmm1, %xmm1, %k15032; AVX512VL-NEXT: vpcompressd %xmm0, %xmm0 {%k1} {z}5033; AVX512VL-NEXT: retq5034 %out = call <4 x i32> @llvm.experimental.vector.compress(<4 x i32> %vec, <4 x i1> %mask, <4 x i32> zeroinitializer)5035 ret <4 x i32> %out5036}5037