1326 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=X64,AVX13; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=X64,AVX24; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=X64,AVX512VL5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512bw,+avx512dq | FileCheck %s --check-prefixes=X64,AVX512VLBWDQ6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq,+avx512cd,+avx512vl | FileCheck %s --check-prefixes=X64,AVX512,AVX512VLCD7; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq,+avx512cd | FileCheck %s --check-prefixes=X64,AVX512,AVX512CD8;9; Just one 32-bit run to make sure we do reasonable things for i64 lzcnt.10; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=X86-AVX11 12define <4 x i64> @testv4i64(<4 x i64> %in) nounwind {13; AVX1-LABEL: testv4i64:14; AVX1: # %bb.0:15; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm216; AVX1-NEXT: vmovq {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]17; AVX1-NEXT: vpshufb %xmm2, %xmm3, %xmm418; AVX1-NEXT: vpsrlw $4, %xmm2, %xmm119; AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]20; AVX1-NEXT: vpand %xmm5, %xmm1, %xmm621; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm122; AVX1-NEXT: vpcmpeqb %xmm1, %xmm6, %xmm723; AVX1-NEXT: vpand %xmm7, %xmm4, %xmm424; AVX1-NEXT: vpshufb %xmm6, %xmm3, %xmm625; AVX1-NEXT: vpaddb %xmm6, %xmm4, %xmm426; AVX1-NEXT: vpcmpeqb %xmm1, %xmm2, %xmm627; AVX1-NEXT: vpsrlw $8, %xmm6, %xmm628; AVX1-NEXT: vpand %xmm6, %xmm4, %xmm629; AVX1-NEXT: vpsrlw $8, %xmm4, %xmm430; AVX1-NEXT: vpaddw %xmm6, %xmm4, %xmm431; AVX1-NEXT: vpcmpeqw %xmm1, %xmm2, %xmm632; AVX1-NEXT: vpsrld $16, %xmm6, %xmm633; AVX1-NEXT: vpand %xmm6, %xmm4, %xmm634; AVX1-NEXT: vpsrld $16, %xmm4, %xmm435; AVX1-NEXT: vpaddd %xmm6, %xmm4, %xmm436; AVX1-NEXT: vpcmpeqd %xmm1, %xmm2, %xmm237; AVX1-NEXT: vpsrlq $32, %xmm2, %xmm238; AVX1-NEXT: vpand %xmm2, %xmm4, %xmm239; AVX1-NEXT: vpsrlq $32, %xmm4, %xmm440; AVX1-NEXT: vpaddq %xmm2, %xmm4, %xmm241; AVX1-NEXT: vpshufb %xmm0, %xmm3, %xmm442; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm643; AVX1-NEXT: vpand %xmm5, %xmm6, %xmm544; AVX1-NEXT: vpcmpeqb %xmm1, %xmm5, %xmm645; AVX1-NEXT: vpand %xmm6, %xmm4, %xmm446; AVX1-NEXT: vpshufb %xmm5, %xmm3, %xmm347; AVX1-NEXT: vpaddb %xmm3, %xmm4, %xmm348; AVX1-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm449; AVX1-NEXT: vpsrlw $8, %xmm4, %xmm450; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm451; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm352; AVX1-NEXT: vpaddw %xmm4, %xmm3, %xmm353; AVX1-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm454; AVX1-NEXT: vpsrld $16, %xmm4, %xmm455; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm456; AVX1-NEXT: vpsrld $16, %xmm3, %xmm357; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm358; AVX1-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm059; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm060; AVX1-NEXT: vpand %xmm0, %xmm3, %xmm061; AVX1-NEXT: vpsrlq $32, %xmm3, %xmm162; AVX1-NEXT: vpaddq %xmm0, %xmm1, %xmm063; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm064; AVX1-NEXT: retq65;66; AVX2-LABEL: testv4i64:67; AVX2: # %bb.0:68; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]69; AVX2-NEXT: # ymm1 = mem[0,1,0,1]70; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm271; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm372; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm373; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm474; AVX2-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm575; AVX2-NEXT: vpand %ymm5, %ymm2, %ymm276; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm177; AVX2-NEXT: vpaddb %ymm1, %ymm2, %ymm178; AVX2-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm279; AVX2-NEXT: vpsrlw $8, %ymm2, %ymm280; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm281; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm182; AVX2-NEXT: vpaddw %ymm2, %ymm1, %ymm183; AVX2-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm284; AVX2-NEXT: vpsrld $16, %ymm2, %ymm285; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm286; AVX2-NEXT: vpsrld $16, %ymm1, %ymm187; AVX2-NEXT: vpaddd %ymm2, %ymm1, %ymm188; AVX2-NEXT: vpcmpeqd %ymm4, %ymm0, %ymm089; AVX2-NEXT: vpsrlq $32, %ymm0, %ymm090; AVX2-NEXT: vpand %ymm0, %ymm1, %ymm091; AVX2-NEXT: vpsrlq $32, %ymm1, %ymm192; AVX2-NEXT: vpaddq %ymm0, %ymm1, %ymm093; AVX2-NEXT: retq94;95; AVX512VL-LABEL: testv4i64:96; AVX512VL: # %bb.0:97; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]98; AVX512VL-NEXT: # ymm1 = mem[0,1,0,1]99; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm2100; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3101; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3102; AVX512VL-NEXT: vpxor %xmm4, %xmm4, %xmm4103; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5104; AVX512VL-NEXT: vpand %ymm5, %ymm2, %ymm2105; AVX512VL-NEXT: vpshufb %ymm3, %ymm1, %ymm1106; AVX512VL-NEXT: vpaddb %ymm1, %ymm2, %ymm1107; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2108; AVX512VL-NEXT: vpsrlw $8, %ymm2, %ymm2109; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm2110; AVX512VL-NEXT: vpsrlw $8, %ymm1, %ymm1111; AVX512VL-NEXT: vpaddw %ymm2, %ymm1, %ymm1112; AVX512VL-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm2113; AVX512VL-NEXT: vpsrld $16, %ymm2, %ymm2114; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm2115; AVX512VL-NEXT: vpsrld $16, %ymm1, %ymm1116; AVX512VL-NEXT: vpaddd %ymm2, %ymm1, %ymm1117; AVX512VL-NEXT: vpcmpeqd %ymm4, %ymm0, %ymm0118; AVX512VL-NEXT: vpsrlq $32, %ymm0, %ymm0119; AVX512VL-NEXT: vpand %ymm0, %ymm1, %ymm0120; AVX512VL-NEXT: vpsrlq $32, %ymm1, %ymm1121; AVX512VL-NEXT: vpaddq %ymm0, %ymm1, %ymm0122; AVX512VL-NEXT: retq123;124; AVX512VLBWDQ-LABEL: testv4i64:125; AVX512VLBWDQ: # %bb.0:126; AVX512VLBWDQ-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]127; AVX512VLBWDQ-NEXT: # ymm1 = mem[0,1,0,1]128; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2129; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm3130; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3131; AVX512VLBWDQ-NEXT: vpxor %xmm4, %xmm4, %xmm4132; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5133; AVX512VLBWDQ-NEXT: vpand %ymm5, %ymm2, %ymm2134; AVX512VLBWDQ-NEXT: vpshufb %ymm3, %ymm1, %ymm1135; AVX512VLBWDQ-NEXT: vpaddb %ymm1, %ymm2, %ymm1136; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2137; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm2, %ymm2138; AVX512VLBWDQ-NEXT: vpand %ymm2, %ymm1, %ymm2139; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm1, %ymm1140; AVX512VLBWDQ-NEXT: vpaddw %ymm2, %ymm1, %ymm1141; AVX512VLBWDQ-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm2142; AVX512VLBWDQ-NEXT: vpsrld $16, %ymm2, %ymm2143; AVX512VLBWDQ-NEXT: vpand %ymm2, %ymm1, %ymm2144; AVX512VLBWDQ-NEXT: vpsrld $16, %ymm1, %ymm1145; AVX512VLBWDQ-NEXT: vpaddd %ymm2, %ymm1, %ymm1146; AVX512VLBWDQ-NEXT: vpcmpeqd %ymm4, %ymm0, %ymm0147; AVX512VLBWDQ-NEXT: vpsrlq $32, %ymm0, %ymm0148; AVX512VLBWDQ-NEXT: vpand %ymm0, %ymm1, %ymm0149; AVX512VLBWDQ-NEXT: vpsrlq $32, %ymm1, %ymm1150; AVX512VLBWDQ-NEXT: vpaddq %ymm0, %ymm1, %ymm0151; AVX512VLBWDQ-NEXT: retq152;153; AVX512VLCD-LABEL: testv4i64:154; AVX512VLCD: # %bb.0:155; AVX512VLCD-NEXT: vplzcntq %ymm0, %ymm0156; AVX512VLCD-NEXT: retq157;158; AVX512CD-LABEL: testv4i64:159; AVX512CD: # %bb.0:160; AVX512CD-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0161; AVX512CD-NEXT: vplzcntq %zmm0, %zmm0162; AVX512CD-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0163; AVX512CD-NEXT: retq164;165; X86-AVX-LABEL: testv4i64:166; X86-AVX: # %bb.0:167; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]168; X86-AVX-NEXT: # ymm1 = mem[0,1,0,1]169; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm2170; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm3171; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3172; X86-AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4173; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5174; X86-AVX-NEXT: vpand %ymm5, %ymm2, %ymm2175; X86-AVX-NEXT: vpshufb %ymm3, %ymm1, %ymm1176; X86-AVX-NEXT: vpaddb %ymm1, %ymm2, %ymm1177; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2178; X86-AVX-NEXT: vpsrlw $8, %ymm2, %ymm2179; X86-AVX-NEXT: vpand %ymm2, %ymm1, %ymm2180; X86-AVX-NEXT: vpsrlw $8, %ymm1, %ymm1181; X86-AVX-NEXT: vpaddw %ymm2, %ymm1, %ymm1182; X86-AVX-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm2183; X86-AVX-NEXT: vpsrld $16, %ymm2, %ymm2184; X86-AVX-NEXT: vpand %ymm2, %ymm1, %ymm2185; X86-AVX-NEXT: vpsrld $16, %ymm1, %ymm1186; X86-AVX-NEXT: vpaddd %ymm2, %ymm1, %ymm1187; X86-AVX-NEXT: vpcmpeqd %ymm4, %ymm0, %ymm0188; X86-AVX-NEXT: vpsrlq $32, %ymm0, %ymm0189; X86-AVX-NEXT: vpand %ymm0, %ymm1, %ymm0190; X86-AVX-NEXT: vpsrlq $32, %ymm1, %ymm1191; X86-AVX-NEXT: vpaddq %ymm0, %ymm1, %ymm0192; X86-AVX-NEXT: retl193 194 %out = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %in, i1 0)195 ret <4 x i64> %out196}197 198define <4 x i64> @testv4i64u(<4 x i64> %in) nounwind {199; AVX1-LABEL: testv4i64u:200; AVX1: # %bb.0:201; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2202; AVX1-NEXT: vmovq {{.*#+}} xmm3 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]203; AVX1-NEXT: vpshufb %xmm2, %xmm3, %xmm4204; AVX1-NEXT: vpsrlw $4, %xmm2, %xmm1205; AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]206; AVX1-NEXT: vpand %xmm5, %xmm1, %xmm6207; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1208; AVX1-NEXT: vpcmpeqb %xmm1, %xmm6, %xmm7209; AVX1-NEXT: vpand %xmm7, %xmm4, %xmm4210; AVX1-NEXT: vpshufb %xmm6, %xmm3, %xmm6211; AVX1-NEXT: vpaddb %xmm6, %xmm4, %xmm4212; AVX1-NEXT: vpcmpeqb %xmm1, %xmm2, %xmm6213; AVX1-NEXT: vpsrlw $8, %xmm6, %xmm6214; AVX1-NEXT: vpand %xmm6, %xmm4, %xmm6215; AVX1-NEXT: vpsrlw $8, %xmm4, %xmm4216; AVX1-NEXT: vpaddw %xmm6, %xmm4, %xmm4217; AVX1-NEXT: vpcmpeqw %xmm1, %xmm2, %xmm6218; AVX1-NEXT: vpsrld $16, %xmm6, %xmm6219; AVX1-NEXT: vpand %xmm6, %xmm4, %xmm6220; AVX1-NEXT: vpsrld $16, %xmm4, %xmm4221; AVX1-NEXT: vpaddd %xmm6, %xmm4, %xmm4222; AVX1-NEXT: vpcmpeqd %xmm1, %xmm2, %xmm2223; AVX1-NEXT: vpsrlq $32, %xmm2, %xmm2224; AVX1-NEXT: vpand %xmm2, %xmm4, %xmm2225; AVX1-NEXT: vpsrlq $32, %xmm4, %xmm4226; AVX1-NEXT: vpaddq %xmm2, %xmm4, %xmm2227; AVX1-NEXT: vpshufb %xmm0, %xmm3, %xmm4228; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm6229; AVX1-NEXT: vpand %xmm5, %xmm6, %xmm5230; AVX1-NEXT: vpcmpeqb %xmm1, %xmm5, %xmm6231; AVX1-NEXT: vpand %xmm6, %xmm4, %xmm4232; AVX1-NEXT: vpshufb %xmm5, %xmm3, %xmm3233; AVX1-NEXT: vpaddb %xmm3, %xmm4, %xmm3234; AVX1-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm4235; AVX1-NEXT: vpsrlw $8, %xmm4, %xmm4236; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm4237; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3238; AVX1-NEXT: vpaddw %xmm4, %xmm3, %xmm3239; AVX1-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm4240; AVX1-NEXT: vpsrld $16, %xmm4, %xmm4241; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm4242; AVX1-NEXT: vpsrld $16, %xmm3, %xmm3243; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3244; AVX1-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0245; AVX1-NEXT: vpsrlq $32, %xmm0, %xmm0246; AVX1-NEXT: vpand %xmm0, %xmm3, %xmm0247; AVX1-NEXT: vpsrlq $32, %xmm3, %xmm1248; AVX1-NEXT: vpaddq %xmm0, %xmm1, %xmm0249; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0250; AVX1-NEXT: retq251;252; AVX2-LABEL: testv4i64u:253; AVX2: # %bb.0:254; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]255; AVX2-NEXT: # ymm1 = mem[0,1,0,1]256; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm2257; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm3258; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3259; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4260; AVX2-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5261; AVX2-NEXT: vpand %ymm5, %ymm2, %ymm2262; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1263; AVX2-NEXT: vpaddb %ymm1, %ymm2, %ymm1264; AVX2-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2265; AVX2-NEXT: vpsrlw $8, %ymm2, %ymm2266; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2267; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1268; AVX2-NEXT: vpaddw %ymm2, %ymm1, %ymm1269; AVX2-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm2270; AVX2-NEXT: vpsrld $16, %ymm2, %ymm2271; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2272; AVX2-NEXT: vpsrld $16, %ymm1, %ymm1273; AVX2-NEXT: vpaddd %ymm2, %ymm1, %ymm1274; AVX2-NEXT: vpcmpeqd %ymm4, %ymm0, %ymm0275; AVX2-NEXT: vpsrlq $32, %ymm0, %ymm0276; AVX2-NEXT: vpand %ymm0, %ymm1, %ymm0277; AVX2-NEXT: vpsrlq $32, %ymm1, %ymm1278; AVX2-NEXT: vpaddq %ymm0, %ymm1, %ymm0279; AVX2-NEXT: retq280;281; AVX512VL-LABEL: testv4i64u:282; AVX512VL: # %bb.0:283; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]284; AVX512VL-NEXT: # ymm1 = mem[0,1,0,1]285; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm2286; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3287; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3288; AVX512VL-NEXT: vpxor %xmm4, %xmm4, %xmm4289; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5290; AVX512VL-NEXT: vpand %ymm5, %ymm2, %ymm2291; AVX512VL-NEXT: vpshufb %ymm3, %ymm1, %ymm1292; AVX512VL-NEXT: vpaddb %ymm1, %ymm2, %ymm1293; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2294; AVX512VL-NEXT: vpsrlw $8, %ymm2, %ymm2295; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm2296; AVX512VL-NEXT: vpsrlw $8, %ymm1, %ymm1297; AVX512VL-NEXT: vpaddw %ymm2, %ymm1, %ymm1298; AVX512VL-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm2299; AVX512VL-NEXT: vpsrld $16, %ymm2, %ymm2300; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm2301; AVX512VL-NEXT: vpsrld $16, %ymm1, %ymm1302; AVX512VL-NEXT: vpaddd %ymm2, %ymm1, %ymm1303; AVX512VL-NEXT: vpcmpeqd %ymm4, %ymm0, %ymm0304; AVX512VL-NEXT: vpsrlq $32, %ymm0, %ymm0305; AVX512VL-NEXT: vpand %ymm0, %ymm1, %ymm0306; AVX512VL-NEXT: vpsrlq $32, %ymm1, %ymm1307; AVX512VL-NEXT: vpaddq %ymm0, %ymm1, %ymm0308; AVX512VL-NEXT: retq309;310; AVX512VLBWDQ-LABEL: testv4i64u:311; AVX512VLBWDQ: # %bb.0:312; AVX512VLBWDQ-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]313; AVX512VLBWDQ-NEXT: # ymm1 = mem[0,1,0,1]314; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2315; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm3316; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3317; AVX512VLBWDQ-NEXT: vpxor %xmm4, %xmm4, %xmm4318; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5319; AVX512VLBWDQ-NEXT: vpand %ymm5, %ymm2, %ymm2320; AVX512VLBWDQ-NEXT: vpshufb %ymm3, %ymm1, %ymm1321; AVX512VLBWDQ-NEXT: vpaddb %ymm1, %ymm2, %ymm1322; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2323; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm2, %ymm2324; AVX512VLBWDQ-NEXT: vpand %ymm2, %ymm1, %ymm2325; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm1, %ymm1326; AVX512VLBWDQ-NEXT: vpaddw %ymm2, %ymm1, %ymm1327; AVX512VLBWDQ-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm2328; AVX512VLBWDQ-NEXT: vpsrld $16, %ymm2, %ymm2329; AVX512VLBWDQ-NEXT: vpand %ymm2, %ymm1, %ymm2330; AVX512VLBWDQ-NEXT: vpsrld $16, %ymm1, %ymm1331; AVX512VLBWDQ-NEXT: vpaddd %ymm2, %ymm1, %ymm1332; AVX512VLBWDQ-NEXT: vpcmpeqd %ymm4, %ymm0, %ymm0333; AVX512VLBWDQ-NEXT: vpsrlq $32, %ymm0, %ymm0334; AVX512VLBWDQ-NEXT: vpand %ymm0, %ymm1, %ymm0335; AVX512VLBWDQ-NEXT: vpsrlq $32, %ymm1, %ymm1336; AVX512VLBWDQ-NEXT: vpaddq %ymm0, %ymm1, %ymm0337; AVX512VLBWDQ-NEXT: retq338;339; AVX512VLCD-LABEL: testv4i64u:340; AVX512VLCD: # %bb.0:341; AVX512VLCD-NEXT: vplzcntq %ymm0, %ymm0342; AVX512VLCD-NEXT: retq343;344; AVX512CD-LABEL: testv4i64u:345; AVX512CD: # %bb.0:346; AVX512CD-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0347; AVX512CD-NEXT: vplzcntq %zmm0, %zmm0348; AVX512CD-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0349; AVX512CD-NEXT: retq350;351; X86-AVX-LABEL: testv4i64u:352; X86-AVX: # %bb.0:353; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]354; X86-AVX-NEXT: # ymm1 = mem[0,1,0,1]355; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm2356; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm3357; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3358; X86-AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4359; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5360; X86-AVX-NEXT: vpand %ymm5, %ymm2, %ymm2361; X86-AVX-NEXT: vpshufb %ymm3, %ymm1, %ymm1362; X86-AVX-NEXT: vpaddb %ymm1, %ymm2, %ymm1363; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2364; X86-AVX-NEXT: vpsrlw $8, %ymm2, %ymm2365; X86-AVX-NEXT: vpand %ymm2, %ymm1, %ymm2366; X86-AVX-NEXT: vpsrlw $8, %ymm1, %ymm1367; X86-AVX-NEXT: vpaddw %ymm2, %ymm1, %ymm1368; X86-AVX-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm2369; X86-AVX-NEXT: vpsrld $16, %ymm2, %ymm2370; X86-AVX-NEXT: vpand %ymm2, %ymm1, %ymm2371; X86-AVX-NEXT: vpsrld $16, %ymm1, %ymm1372; X86-AVX-NEXT: vpaddd %ymm2, %ymm1, %ymm1373; X86-AVX-NEXT: vpcmpeqd %ymm4, %ymm0, %ymm0374; X86-AVX-NEXT: vpsrlq $32, %ymm0, %ymm0375; X86-AVX-NEXT: vpand %ymm0, %ymm1, %ymm0376; X86-AVX-NEXT: vpsrlq $32, %ymm1, %ymm1377; X86-AVX-NEXT: vpaddq %ymm0, %ymm1, %ymm0378; X86-AVX-NEXT: retl379 380 %out = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %in, i1 -1)381 ret <4 x i64> %out382}383 384define <8 x i32> @testv8i32(<8 x i32> %in) nounwind {385; AVX1-LABEL: testv8i32:386; AVX1: # %bb.0:387; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1388; AVX1-NEXT: vmovq {{.*#+}} xmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]389; AVX1-NEXT: vpshufb %xmm1, %xmm2, %xmm3390; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm4391; AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]392; AVX1-NEXT: vpand %xmm5, %xmm4, %xmm4393; AVX1-NEXT: vpxor %xmm6, %xmm6, %xmm6394; AVX1-NEXT: vpcmpeqb %xmm6, %xmm4, %xmm7395; AVX1-NEXT: vpand %xmm7, %xmm3, %xmm3396; AVX1-NEXT: vpshufb %xmm4, %xmm2, %xmm4397; AVX1-NEXT: vpaddb %xmm4, %xmm3, %xmm3398; AVX1-NEXT: vpcmpeqb %xmm6, %xmm1, %xmm4399; AVX1-NEXT: vpsrlw $8, %xmm4, %xmm4400; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm4401; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3402; AVX1-NEXT: vpaddw %xmm4, %xmm3, %xmm3403; AVX1-NEXT: vpcmpeqw %xmm6, %xmm1, %xmm1404; AVX1-NEXT: vpsrld $16, %xmm1, %xmm1405; AVX1-NEXT: vpand %xmm1, %xmm3, %xmm1406; AVX1-NEXT: vpsrld $16, %xmm3, %xmm3407; AVX1-NEXT: vpaddd %xmm1, %xmm3, %xmm1408; AVX1-NEXT: vpshufb %xmm0, %xmm2, %xmm3409; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm4410; AVX1-NEXT: vpand %xmm5, %xmm4, %xmm4411; AVX1-NEXT: vpcmpeqb %xmm6, %xmm4, %xmm5412; AVX1-NEXT: vpand %xmm5, %xmm3, %xmm3413; AVX1-NEXT: vpshufb %xmm4, %xmm2, %xmm2414; AVX1-NEXT: vpaddb %xmm2, %xmm3, %xmm2415; AVX1-NEXT: vpcmpeqb %xmm6, %xmm0, %xmm3416; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3417; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm3418; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2419; AVX1-NEXT: vpaddw %xmm3, %xmm2, %xmm2420; AVX1-NEXT: vpcmpeqw %xmm6, %xmm0, %xmm0421; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0422; AVX1-NEXT: vpand %xmm0, %xmm2, %xmm0423; AVX1-NEXT: vpsrld $16, %xmm2, %xmm2424; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0425; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0426; AVX1-NEXT: retq427;428; AVX2-LABEL: testv8i32:429; AVX2: # %bb.0:430; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]431; AVX2-NEXT: # ymm1 = mem[0,1,0,1]432; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm2433; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm3434; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3435; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4436; AVX2-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5437; AVX2-NEXT: vpand %ymm5, %ymm2, %ymm2438; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1439; AVX2-NEXT: vpaddb %ymm1, %ymm2, %ymm1440; AVX2-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2441; AVX2-NEXT: vpsrlw $8, %ymm2, %ymm2442; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2443; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1444; AVX2-NEXT: vpaddw %ymm2, %ymm1, %ymm1445; AVX2-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0446; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0447; AVX2-NEXT: vpand %ymm0, %ymm1, %ymm0448; AVX2-NEXT: vpsrld $16, %ymm1, %ymm1449; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0450; AVX2-NEXT: retq451;452; AVX512VL-LABEL: testv8i32:453; AVX512VL: # %bb.0:454; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]455; AVX512VL-NEXT: # ymm1 = mem[0,1,0,1]456; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm2457; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3458; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3459; AVX512VL-NEXT: vpxor %xmm4, %xmm4, %xmm4460; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5461; AVX512VL-NEXT: vpand %ymm5, %ymm2, %ymm2462; AVX512VL-NEXT: vpshufb %ymm3, %ymm1, %ymm1463; AVX512VL-NEXT: vpaddb %ymm1, %ymm2, %ymm1464; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2465; AVX512VL-NEXT: vpsrlw $8, %ymm2, %ymm2466; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm2467; AVX512VL-NEXT: vpsrlw $8, %ymm1, %ymm1468; AVX512VL-NEXT: vpaddw %ymm2, %ymm1, %ymm1469; AVX512VL-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0470; AVX512VL-NEXT: vpsrld $16, %ymm0, %ymm0471; AVX512VL-NEXT: vpand %ymm0, %ymm1, %ymm0472; AVX512VL-NEXT: vpsrld $16, %ymm1, %ymm1473; AVX512VL-NEXT: vpaddd %ymm0, %ymm1, %ymm0474; AVX512VL-NEXT: retq475;476; AVX512VLBWDQ-LABEL: testv8i32:477; AVX512VLBWDQ: # %bb.0:478; AVX512VLBWDQ-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]479; AVX512VLBWDQ-NEXT: # ymm1 = mem[0,1,0,1]480; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2481; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm3482; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3483; AVX512VLBWDQ-NEXT: vpxor %xmm4, %xmm4, %xmm4484; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5485; AVX512VLBWDQ-NEXT: vpand %ymm5, %ymm2, %ymm2486; AVX512VLBWDQ-NEXT: vpshufb %ymm3, %ymm1, %ymm1487; AVX512VLBWDQ-NEXT: vpaddb %ymm1, %ymm2, %ymm1488; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2489; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm2, %ymm2490; AVX512VLBWDQ-NEXT: vpand %ymm2, %ymm1, %ymm2491; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm1, %ymm1492; AVX512VLBWDQ-NEXT: vpaddw %ymm2, %ymm1, %ymm1493; AVX512VLBWDQ-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0494; AVX512VLBWDQ-NEXT: vpsrld $16, %ymm0, %ymm0495; AVX512VLBWDQ-NEXT: vpand %ymm0, %ymm1, %ymm0496; AVX512VLBWDQ-NEXT: vpsrld $16, %ymm1, %ymm1497; AVX512VLBWDQ-NEXT: vpaddd %ymm0, %ymm1, %ymm0498; AVX512VLBWDQ-NEXT: retq499;500; AVX512VLCD-LABEL: testv8i32:501; AVX512VLCD: # %bb.0:502; AVX512VLCD-NEXT: vplzcntd %ymm0, %ymm0503; AVX512VLCD-NEXT: retq504;505; AVX512CD-LABEL: testv8i32:506; AVX512CD: # %bb.0:507; AVX512CD-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0508; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0509; AVX512CD-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0510; AVX512CD-NEXT: retq511;512; X86-AVX-LABEL: testv8i32:513; X86-AVX: # %bb.0:514; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]515; X86-AVX-NEXT: # ymm1 = mem[0,1,0,1]516; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm2517; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm3518; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3519; X86-AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4520; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5521; X86-AVX-NEXT: vpand %ymm5, %ymm2, %ymm2522; X86-AVX-NEXT: vpshufb %ymm3, %ymm1, %ymm1523; X86-AVX-NEXT: vpaddb %ymm1, %ymm2, %ymm1524; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2525; X86-AVX-NEXT: vpsrlw $8, %ymm2, %ymm2526; X86-AVX-NEXT: vpand %ymm2, %ymm1, %ymm2527; X86-AVX-NEXT: vpsrlw $8, %ymm1, %ymm1528; X86-AVX-NEXT: vpaddw %ymm2, %ymm1, %ymm1529; X86-AVX-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0530; X86-AVX-NEXT: vpsrld $16, %ymm0, %ymm0531; X86-AVX-NEXT: vpand %ymm0, %ymm1, %ymm0532; X86-AVX-NEXT: vpsrld $16, %ymm1, %ymm1533; X86-AVX-NEXT: vpaddd %ymm0, %ymm1, %ymm0534; X86-AVX-NEXT: retl535 536 %out = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %in, i1 0)537 ret <8 x i32> %out538}539 540define <8 x i32> @testv8i32u(<8 x i32> %in) nounwind {541; AVX1-LABEL: testv8i32u:542; AVX1: # %bb.0:543; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1544; AVX1-NEXT: vmovq {{.*#+}} xmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]545; AVX1-NEXT: vpshufb %xmm1, %xmm2, %xmm3546; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm4547; AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]548; AVX1-NEXT: vpand %xmm5, %xmm4, %xmm4549; AVX1-NEXT: vpxor %xmm6, %xmm6, %xmm6550; AVX1-NEXT: vpcmpeqb %xmm6, %xmm4, %xmm7551; AVX1-NEXT: vpand %xmm7, %xmm3, %xmm3552; AVX1-NEXT: vpshufb %xmm4, %xmm2, %xmm4553; AVX1-NEXT: vpaddb %xmm4, %xmm3, %xmm3554; AVX1-NEXT: vpcmpeqb %xmm6, %xmm1, %xmm4555; AVX1-NEXT: vpsrlw $8, %xmm4, %xmm4556; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm4557; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3558; AVX1-NEXT: vpaddw %xmm4, %xmm3, %xmm3559; AVX1-NEXT: vpcmpeqw %xmm6, %xmm1, %xmm1560; AVX1-NEXT: vpsrld $16, %xmm1, %xmm1561; AVX1-NEXT: vpand %xmm1, %xmm3, %xmm1562; AVX1-NEXT: vpsrld $16, %xmm3, %xmm3563; AVX1-NEXT: vpaddd %xmm1, %xmm3, %xmm1564; AVX1-NEXT: vpshufb %xmm0, %xmm2, %xmm3565; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm4566; AVX1-NEXT: vpand %xmm5, %xmm4, %xmm4567; AVX1-NEXT: vpcmpeqb %xmm6, %xmm4, %xmm5568; AVX1-NEXT: vpand %xmm5, %xmm3, %xmm3569; AVX1-NEXT: vpshufb %xmm4, %xmm2, %xmm2570; AVX1-NEXT: vpaddb %xmm2, %xmm3, %xmm2571; AVX1-NEXT: vpcmpeqb %xmm6, %xmm0, %xmm3572; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3573; AVX1-NEXT: vpand %xmm3, %xmm2, %xmm3574; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2575; AVX1-NEXT: vpaddw %xmm3, %xmm2, %xmm2576; AVX1-NEXT: vpcmpeqw %xmm6, %xmm0, %xmm0577; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0578; AVX1-NEXT: vpand %xmm0, %xmm2, %xmm0579; AVX1-NEXT: vpsrld $16, %xmm2, %xmm2580; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0581; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0582; AVX1-NEXT: retq583;584; AVX2-LABEL: testv8i32u:585; AVX2: # %bb.0:586; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]587; AVX2-NEXT: # ymm1 = mem[0,1,0,1]588; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm2589; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm3590; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3591; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4592; AVX2-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5593; AVX2-NEXT: vpand %ymm5, %ymm2, %ymm2594; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1595; AVX2-NEXT: vpaddb %ymm1, %ymm2, %ymm1596; AVX2-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2597; AVX2-NEXT: vpsrlw $8, %ymm2, %ymm2598; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2599; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1600; AVX2-NEXT: vpaddw %ymm2, %ymm1, %ymm1601; AVX2-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0602; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0603; AVX2-NEXT: vpand %ymm0, %ymm1, %ymm0604; AVX2-NEXT: vpsrld $16, %ymm1, %ymm1605; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0606; AVX2-NEXT: retq607;608; AVX512VL-LABEL: testv8i32u:609; AVX512VL: # %bb.0:610; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]611; AVX512VL-NEXT: # ymm1 = mem[0,1,0,1]612; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm2613; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3614; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3615; AVX512VL-NEXT: vpxor %xmm4, %xmm4, %xmm4616; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5617; AVX512VL-NEXT: vpand %ymm5, %ymm2, %ymm2618; AVX512VL-NEXT: vpshufb %ymm3, %ymm1, %ymm1619; AVX512VL-NEXT: vpaddb %ymm1, %ymm2, %ymm1620; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2621; AVX512VL-NEXT: vpsrlw $8, %ymm2, %ymm2622; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm2623; AVX512VL-NEXT: vpsrlw $8, %ymm1, %ymm1624; AVX512VL-NEXT: vpaddw %ymm2, %ymm1, %ymm1625; AVX512VL-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0626; AVX512VL-NEXT: vpsrld $16, %ymm0, %ymm0627; AVX512VL-NEXT: vpand %ymm0, %ymm1, %ymm0628; AVX512VL-NEXT: vpsrld $16, %ymm1, %ymm1629; AVX512VL-NEXT: vpaddd %ymm0, %ymm1, %ymm0630; AVX512VL-NEXT: retq631;632; AVX512VLBWDQ-LABEL: testv8i32u:633; AVX512VLBWDQ: # %bb.0:634; AVX512VLBWDQ-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]635; AVX512VLBWDQ-NEXT: # ymm1 = mem[0,1,0,1]636; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2637; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm3638; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3639; AVX512VLBWDQ-NEXT: vpxor %xmm4, %xmm4, %xmm4640; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5641; AVX512VLBWDQ-NEXT: vpand %ymm5, %ymm2, %ymm2642; AVX512VLBWDQ-NEXT: vpshufb %ymm3, %ymm1, %ymm1643; AVX512VLBWDQ-NEXT: vpaddb %ymm1, %ymm2, %ymm1644; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2645; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm2, %ymm2646; AVX512VLBWDQ-NEXT: vpand %ymm2, %ymm1, %ymm2647; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm1, %ymm1648; AVX512VLBWDQ-NEXT: vpaddw %ymm2, %ymm1, %ymm1649; AVX512VLBWDQ-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0650; AVX512VLBWDQ-NEXT: vpsrld $16, %ymm0, %ymm0651; AVX512VLBWDQ-NEXT: vpand %ymm0, %ymm1, %ymm0652; AVX512VLBWDQ-NEXT: vpsrld $16, %ymm1, %ymm1653; AVX512VLBWDQ-NEXT: vpaddd %ymm0, %ymm1, %ymm0654; AVX512VLBWDQ-NEXT: retq655;656; AVX512VLCD-LABEL: testv8i32u:657; AVX512VLCD: # %bb.0:658; AVX512VLCD-NEXT: vplzcntd %ymm0, %ymm0659; AVX512VLCD-NEXT: retq660;661; AVX512CD-LABEL: testv8i32u:662; AVX512CD: # %bb.0:663; AVX512CD-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0664; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0665; AVX512CD-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0666; AVX512CD-NEXT: retq667;668; X86-AVX-LABEL: testv8i32u:669; X86-AVX: # %bb.0:670; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]671; X86-AVX-NEXT: # ymm1 = mem[0,1,0,1]672; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm2673; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm3674; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3675; X86-AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4676; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5677; X86-AVX-NEXT: vpand %ymm5, %ymm2, %ymm2678; X86-AVX-NEXT: vpshufb %ymm3, %ymm1, %ymm1679; X86-AVX-NEXT: vpaddb %ymm1, %ymm2, %ymm1680; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2681; X86-AVX-NEXT: vpsrlw $8, %ymm2, %ymm2682; X86-AVX-NEXT: vpand %ymm2, %ymm1, %ymm2683; X86-AVX-NEXT: vpsrlw $8, %ymm1, %ymm1684; X86-AVX-NEXT: vpaddw %ymm2, %ymm1, %ymm1685; X86-AVX-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0686; X86-AVX-NEXT: vpsrld $16, %ymm0, %ymm0687; X86-AVX-NEXT: vpand %ymm0, %ymm1, %ymm0688; X86-AVX-NEXT: vpsrld $16, %ymm1, %ymm1689; X86-AVX-NEXT: vpaddd %ymm0, %ymm1, %ymm0690; X86-AVX-NEXT: retl691 692 %out = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %in, i1 -1)693 ret <8 x i32> %out694}695 696define <16 x i16> @testv16i16(<16 x i16> %in) nounwind {697; AVX1-LABEL: testv16i16:698; AVX1: # %bb.0:699; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1700; AVX1-NEXT: vmovq {{.*#+}} xmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]701; AVX1-NEXT: vpshufb %xmm1, %xmm2, %xmm3702; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm4703; AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]704; AVX1-NEXT: vpand %xmm5, %xmm4, %xmm4705; AVX1-NEXT: vpxor %xmm6, %xmm6, %xmm6706; AVX1-NEXT: vpcmpeqb %xmm6, %xmm4, %xmm7707; AVX1-NEXT: vpand %xmm7, %xmm3, %xmm3708; AVX1-NEXT: vpshufb %xmm4, %xmm2, %xmm4709; AVX1-NEXT: vpaddb %xmm4, %xmm3, %xmm3710; AVX1-NEXT: vpcmpeqb %xmm6, %xmm1, %xmm1711; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1712; AVX1-NEXT: vpand %xmm1, %xmm3, %xmm1713; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3714; AVX1-NEXT: vpaddw %xmm1, %xmm3, %xmm1715; AVX1-NEXT: vpshufb %xmm0, %xmm2, %xmm3716; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm4717; AVX1-NEXT: vpand %xmm5, %xmm4, %xmm4718; AVX1-NEXT: vpcmpeqb %xmm6, %xmm4, %xmm5719; AVX1-NEXT: vpand %xmm5, %xmm3, %xmm3720; AVX1-NEXT: vpshufb %xmm4, %xmm2, %xmm2721; AVX1-NEXT: vpaddb %xmm2, %xmm3, %xmm2722; AVX1-NEXT: vpcmpeqb %xmm6, %xmm0, %xmm0723; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0724; AVX1-NEXT: vpand %xmm0, %xmm2, %xmm0725; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2726; AVX1-NEXT: vpaddw %xmm0, %xmm2, %xmm0727; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0728; AVX1-NEXT: retq729;730; AVX2-LABEL: testv16i16:731; AVX2: # %bb.0:732; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]733; AVX2-NEXT: # ymm1 = mem[0,1,0,1]734; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm2735; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm3736; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3737; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4738; AVX2-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5739; AVX2-NEXT: vpand %ymm5, %ymm2, %ymm2740; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1741; AVX2-NEXT: vpaddb %ymm1, %ymm2, %ymm1742; AVX2-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0743; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0744; AVX2-NEXT: vpand %ymm0, %ymm1, %ymm0745; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1746; AVX2-NEXT: vpaddw %ymm0, %ymm1, %ymm0747; AVX2-NEXT: retq748;749; AVX512VL-LABEL: testv16i16:750; AVX512VL: # %bb.0:751; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]752; AVX512VL-NEXT: # ymm1 = mem[0,1,0,1]753; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm2754; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3755; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3756; AVX512VL-NEXT: vpxor %xmm4, %xmm4, %xmm4757; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5758; AVX512VL-NEXT: vpand %ymm5, %ymm2, %ymm2759; AVX512VL-NEXT: vpshufb %ymm3, %ymm1, %ymm1760; AVX512VL-NEXT: vpaddb %ymm1, %ymm2, %ymm1761; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0762; AVX512VL-NEXT: vpsrlw $8, %ymm0, %ymm0763; AVX512VL-NEXT: vpand %ymm0, %ymm1, %ymm0764; AVX512VL-NEXT: vpsrlw $8, %ymm1, %ymm1765; AVX512VL-NEXT: vpaddw %ymm0, %ymm1, %ymm0766; AVX512VL-NEXT: retq767;768; AVX512VLBWDQ-LABEL: testv16i16:769; AVX512VLBWDQ: # %bb.0:770; AVX512VLBWDQ-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]771; AVX512VLBWDQ-NEXT: # ymm1 = mem[0,1,0,1]772; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2773; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm3774; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3775; AVX512VLBWDQ-NEXT: vpxor %xmm4, %xmm4, %xmm4776; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5777; AVX512VLBWDQ-NEXT: vpand %ymm5, %ymm2, %ymm2778; AVX512VLBWDQ-NEXT: vpshufb %ymm3, %ymm1, %ymm1779; AVX512VLBWDQ-NEXT: vpaddb %ymm1, %ymm2, %ymm1780; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0781; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm0, %ymm0782; AVX512VLBWDQ-NEXT: vpand %ymm0, %ymm1, %ymm0783; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm1, %ymm1784; AVX512VLBWDQ-NEXT: vpaddw %ymm0, %ymm1, %ymm0785; AVX512VLBWDQ-NEXT: retq786;787; AVX512-LABEL: testv16i16:788; AVX512: # %bb.0:789; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero790; AVX512-NEXT: vplzcntd %zmm0, %zmm0791; AVX512-NEXT: vpmovdw %zmm0, %ymm0792; AVX512-NEXT: vpsubw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0793; AVX512-NEXT: retq794;795; X86-AVX-LABEL: testv16i16:796; X86-AVX: # %bb.0:797; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]798; X86-AVX-NEXT: # ymm1 = mem[0,1,0,1]799; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm2800; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm3801; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3802; X86-AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4803; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5804; X86-AVX-NEXT: vpand %ymm5, %ymm2, %ymm2805; X86-AVX-NEXT: vpshufb %ymm3, %ymm1, %ymm1806; X86-AVX-NEXT: vpaddb %ymm1, %ymm2, %ymm1807; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0808; X86-AVX-NEXT: vpsrlw $8, %ymm0, %ymm0809; X86-AVX-NEXT: vpand %ymm0, %ymm1, %ymm0810; X86-AVX-NEXT: vpsrlw $8, %ymm1, %ymm1811; X86-AVX-NEXT: vpaddw %ymm0, %ymm1, %ymm0812; X86-AVX-NEXT: retl813 %out = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> %in, i1 0)814 ret <16 x i16> %out815}816 817define <16 x i16> @testv16i16u(<16 x i16> %in) nounwind {818; AVX1-LABEL: testv16i16u:819; AVX1: # %bb.0:820; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1821; AVX1-NEXT: vmovq {{.*#+}} xmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]822; AVX1-NEXT: vpshufb %xmm1, %xmm2, %xmm3823; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm4824; AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]825; AVX1-NEXT: vpand %xmm5, %xmm4, %xmm4826; AVX1-NEXT: vpxor %xmm6, %xmm6, %xmm6827; AVX1-NEXT: vpcmpeqb %xmm6, %xmm4, %xmm7828; AVX1-NEXT: vpand %xmm7, %xmm3, %xmm3829; AVX1-NEXT: vpshufb %xmm4, %xmm2, %xmm4830; AVX1-NEXT: vpaddb %xmm4, %xmm3, %xmm3831; AVX1-NEXT: vpcmpeqb %xmm6, %xmm1, %xmm1832; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1833; AVX1-NEXT: vpand %xmm1, %xmm3, %xmm1834; AVX1-NEXT: vpsrlw $8, %xmm3, %xmm3835; AVX1-NEXT: vpaddw %xmm1, %xmm3, %xmm1836; AVX1-NEXT: vpshufb %xmm0, %xmm2, %xmm3837; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm4838; AVX1-NEXT: vpand %xmm5, %xmm4, %xmm4839; AVX1-NEXT: vpcmpeqb %xmm6, %xmm4, %xmm5840; AVX1-NEXT: vpand %xmm5, %xmm3, %xmm3841; AVX1-NEXT: vpshufb %xmm4, %xmm2, %xmm2842; AVX1-NEXT: vpaddb %xmm2, %xmm3, %xmm2843; AVX1-NEXT: vpcmpeqb %xmm6, %xmm0, %xmm0844; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0845; AVX1-NEXT: vpand %xmm0, %xmm2, %xmm0846; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2847; AVX1-NEXT: vpaddw %xmm0, %xmm2, %xmm0848; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0849; AVX1-NEXT: retq850;851; AVX2-LABEL: testv16i16u:852; AVX2: # %bb.0:853; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]854; AVX2-NEXT: # ymm1 = mem[0,1,0,1]855; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm2856; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm3857; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3858; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4859; AVX2-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5860; AVX2-NEXT: vpand %ymm5, %ymm2, %ymm2861; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1862; AVX2-NEXT: vpaddb %ymm1, %ymm2, %ymm1863; AVX2-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0864; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0865; AVX2-NEXT: vpand %ymm0, %ymm1, %ymm0866; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1867; AVX2-NEXT: vpaddw %ymm0, %ymm1, %ymm0868; AVX2-NEXT: retq869;870; AVX512VL-LABEL: testv16i16u:871; AVX512VL: # %bb.0:872; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]873; AVX512VL-NEXT: # ymm1 = mem[0,1,0,1]874; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm2875; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3876; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3877; AVX512VL-NEXT: vpxor %xmm4, %xmm4, %xmm4878; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5879; AVX512VL-NEXT: vpand %ymm5, %ymm2, %ymm2880; AVX512VL-NEXT: vpshufb %ymm3, %ymm1, %ymm1881; AVX512VL-NEXT: vpaddb %ymm1, %ymm2, %ymm1882; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0883; AVX512VL-NEXT: vpsrlw $8, %ymm0, %ymm0884; AVX512VL-NEXT: vpand %ymm0, %ymm1, %ymm0885; AVX512VL-NEXT: vpsrlw $8, %ymm1, %ymm1886; AVX512VL-NEXT: vpaddw %ymm0, %ymm1, %ymm0887; AVX512VL-NEXT: retq888;889; AVX512VLBWDQ-LABEL: testv16i16u:890; AVX512VLBWDQ: # %bb.0:891; AVX512VLBWDQ-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]892; AVX512VLBWDQ-NEXT: # ymm1 = mem[0,1,0,1]893; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2894; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm3895; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3896; AVX512VLBWDQ-NEXT: vpxor %xmm4, %xmm4, %xmm4897; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5898; AVX512VLBWDQ-NEXT: vpand %ymm5, %ymm2, %ymm2899; AVX512VLBWDQ-NEXT: vpshufb %ymm3, %ymm1, %ymm1900; AVX512VLBWDQ-NEXT: vpaddb %ymm1, %ymm2, %ymm1901; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0902; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm0, %ymm0903; AVX512VLBWDQ-NEXT: vpand %ymm0, %ymm1, %ymm0904; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm1, %ymm1905; AVX512VLBWDQ-NEXT: vpaddw %ymm0, %ymm1, %ymm0906; AVX512VLBWDQ-NEXT: retq907;908; AVX512-LABEL: testv16i16u:909; AVX512: # %bb.0:910; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero911; AVX512-NEXT: vplzcntd %zmm0, %zmm0912; AVX512-NEXT: vpmovdw %zmm0, %ymm0913; AVX512-NEXT: vpsubw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0914; AVX512-NEXT: retq915;916; X86-AVX-LABEL: testv16i16u:917; X86-AVX: # %bb.0:918; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]919; X86-AVX-NEXT: # ymm1 = mem[0,1,0,1]920; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm2921; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm3922; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3923; X86-AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4924; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5925; X86-AVX-NEXT: vpand %ymm5, %ymm2, %ymm2926; X86-AVX-NEXT: vpshufb %ymm3, %ymm1, %ymm1927; X86-AVX-NEXT: vpaddb %ymm1, %ymm2, %ymm1928; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0929; X86-AVX-NEXT: vpsrlw $8, %ymm0, %ymm0930; X86-AVX-NEXT: vpand %ymm0, %ymm1, %ymm0931; X86-AVX-NEXT: vpsrlw $8, %ymm1, %ymm1932; X86-AVX-NEXT: vpaddw %ymm0, %ymm1, %ymm0933; X86-AVX-NEXT: retl934 %out = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> %in, i1 -1)935 ret <16 x i16> %out936}937 938define <32 x i8> @testv32i8(<32 x i8> %in) nounwind {939; AVX1-LABEL: testv32i8:940; AVX1: # %bb.0:941; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1942; AVX1-NEXT: vmovq {{.*#+}} xmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]943; AVX1-NEXT: vpshufb %xmm1, %xmm2, %xmm3944; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1945; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]946; AVX1-NEXT: vpand %xmm4, %xmm1, %xmm1947; AVX1-NEXT: vpxor %xmm5, %xmm5, %xmm5948; AVX1-NEXT: vpcmpeqb %xmm5, %xmm1, %xmm6949; AVX1-NEXT: vpand %xmm6, %xmm3, %xmm3950; AVX1-NEXT: vpshufb %xmm1, %xmm2, %xmm1951; AVX1-NEXT: vpaddb %xmm1, %xmm3, %xmm1952; AVX1-NEXT: vpshufb %xmm0, %xmm2, %xmm3953; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0954; AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0955; AVX1-NEXT: vpcmpeqb %xmm5, %xmm0, %xmm4956; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm3957; AVX1-NEXT: vpshufb %xmm0, %xmm2, %xmm0958; AVX1-NEXT: vpaddb %xmm0, %xmm3, %xmm0959; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0960; AVX1-NEXT: retq961;962; AVX2-LABEL: testv32i8:963; AVX2: # %bb.0:964; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]965; AVX2-NEXT: # ymm1 = mem[0,1,0,1]966; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm2967; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0968; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0969; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3970; AVX2-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm3971; AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2972; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm0973; AVX2-NEXT: vpaddb %ymm0, %ymm2, %ymm0974; AVX2-NEXT: retq975;976; AVX512VL-LABEL: testv32i8:977; AVX512VL: # %bb.0:978; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]979; AVX512VL-NEXT: # ymm1 = mem[0,1,0,1]980; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm2981; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0982; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0983; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3984; AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm3985; AVX512VL-NEXT: vpand %ymm3, %ymm2, %ymm2986; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm0987; AVX512VL-NEXT: vpaddb %ymm0, %ymm2, %ymm0988; AVX512VL-NEXT: retq989;990; AVX512VLBWDQ-LABEL: testv32i8:991; AVX512VLBWDQ: # %bb.0:992; AVX512VLBWDQ-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]993; AVX512VLBWDQ-NEXT: # ymm1 = mem[0,1,0,1]994; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2995; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm0996; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0997; AVX512VLBWDQ-NEXT: vpxor %xmm3, %xmm3, %xmm3998; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm3999; AVX512VLBWDQ-NEXT: vpand %ymm3, %ymm2, %ymm21000; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm01001; AVX512VLBWDQ-NEXT: vpaddb %ymm0, %ymm2, %ymm01002; AVX512VLBWDQ-NEXT: retq1003;1004; AVX512-LABEL: testv32i8:1005; AVX512: # %bb.0:1006; AVX512-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero1007; AVX512-NEXT: vplzcntd %zmm1, %zmm11008; AVX512-NEXT: vpmovdb %zmm1, %xmm11009; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm01010; AVX512-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero1011; AVX512-NEXT: vplzcntd %zmm0, %zmm01012; AVX512-NEXT: vpmovdb %zmm0, %xmm01013; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm01014; AVX512-NEXT: vpsubb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01015; AVX512-NEXT: retq1016;1017; X86-AVX-LABEL: testv32i8:1018; X86-AVX: # %bb.0:1019; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]1020; X86-AVX-NEXT: # ymm1 = mem[0,1,0,1]1021; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm21022; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm01023; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm01024; X86-AVX-NEXT: vpxor %xmm3, %xmm3, %xmm31025; X86-AVX-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm31026; X86-AVX-NEXT: vpand %ymm3, %ymm2, %ymm21027; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm01028; X86-AVX-NEXT: vpaddb %ymm0, %ymm2, %ymm01029; X86-AVX-NEXT: retl1030 %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %in, i1 0)1031 ret <32 x i8> %out1032}1033 1034define <32 x i8> @testv32i8u(<32 x i8> %in) nounwind {1035; AVX1-LABEL: testv32i8u:1036; AVX1: # %bb.0:1037; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm11038; AVX1-NEXT: vmovq {{.*#+}} xmm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]1039; AVX1-NEXT: vpshufb %xmm1, %xmm2, %xmm31040; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm11041; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]1042; AVX1-NEXT: vpand %xmm4, %xmm1, %xmm11043; AVX1-NEXT: vpxor %xmm5, %xmm5, %xmm51044; AVX1-NEXT: vpcmpeqb %xmm5, %xmm1, %xmm61045; AVX1-NEXT: vpand %xmm6, %xmm3, %xmm31046; AVX1-NEXT: vpshufb %xmm1, %xmm2, %xmm11047; AVX1-NEXT: vpaddb %xmm1, %xmm3, %xmm11048; AVX1-NEXT: vpshufb %xmm0, %xmm2, %xmm31049; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm01050; AVX1-NEXT: vpand %xmm4, %xmm0, %xmm01051; AVX1-NEXT: vpcmpeqb %xmm5, %xmm0, %xmm41052; AVX1-NEXT: vpand %xmm4, %xmm3, %xmm31053; AVX1-NEXT: vpshufb %xmm0, %xmm2, %xmm01054; AVX1-NEXT: vpaddb %xmm0, %xmm3, %xmm01055; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm01056; AVX1-NEXT: retq1057;1058; AVX2-LABEL: testv32i8u:1059; AVX2: # %bb.0:1060; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]1061; AVX2-NEXT: # ymm1 = mem[0,1,0,1]1062; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm21063; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm01064; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01065; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm31066; AVX2-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm31067; AVX2-NEXT: vpand %ymm3, %ymm2, %ymm21068; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm01069; AVX2-NEXT: vpaddb %ymm0, %ymm2, %ymm01070; AVX2-NEXT: retq1071;1072; AVX512VL-LABEL: testv32i8u:1073; AVX512VL: # %bb.0:1074; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]1075; AVX512VL-NEXT: # ymm1 = mem[0,1,0,1]1076; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm21077; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm01078; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm01079; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm31080; AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm31081; AVX512VL-NEXT: vpand %ymm3, %ymm2, %ymm21082; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm01083; AVX512VL-NEXT: vpaddb %ymm0, %ymm2, %ymm01084; AVX512VL-NEXT: retq1085;1086; AVX512VLBWDQ-LABEL: testv32i8u:1087; AVX512VLBWDQ: # %bb.0:1088; AVX512VLBWDQ-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]1089; AVX512VLBWDQ-NEXT: # ymm1 = mem[0,1,0,1]1090; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm21091; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm01092; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm01093; AVX512VLBWDQ-NEXT: vpxor %xmm3, %xmm3, %xmm31094; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm31095; AVX512VLBWDQ-NEXT: vpand %ymm3, %ymm2, %ymm21096; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm01097; AVX512VLBWDQ-NEXT: vpaddb %ymm0, %ymm2, %ymm01098; AVX512VLBWDQ-NEXT: retq1099;1100; AVX512-LABEL: testv32i8u:1101; AVX512: # %bb.0:1102; AVX512-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero1103; AVX512-NEXT: vplzcntd %zmm1, %zmm11104; AVX512-NEXT: vpmovdb %zmm1, %xmm11105; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm01106; AVX512-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero1107; AVX512-NEXT: vplzcntd %zmm0, %zmm01108; AVX512-NEXT: vpmovdb %zmm0, %xmm01109; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm01110; AVX512-NEXT: vpsubb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01111; AVX512-NEXT: retq1112;1113; X86-AVX-LABEL: testv32i8u:1114; X86-AVX: # %bb.0:1115; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]1116; X86-AVX-NEXT: # ymm1 = mem[0,1,0,1]1117; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm21118; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm01119; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm01120; X86-AVX-NEXT: vpxor %xmm3, %xmm3, %xmm31121; X86-AVX-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm31122; X86-AVX-NEXT: vpand %ymm3, %ymm2, %ymm21123; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm01124; X86-AVX-NEXT: vpaddb %ymm0, %ymm2, %ymm01125; X86-AVX-NEXT: retl1126 %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %in, i1 -1)1127 ret <32 x i8> %out1128}1129 1130define <4 x i64> @foldv4i64() nounwind {1131; AVX1-LABEL: foldv4i64:1132; AVX1: # %bb.0:1133; AVX1-NEXT: vmovaps {{.*#+}} ymm0 = [55,0,64,56]1134; AVX1-NEXT: retq1135;1136; AVX2-LABEL: foldv4i64:1137; AVX2: # %bb.0:1138; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [55,0,64,56]1139; AVX2-NEXT: retq1140;1141; AVX512VL-LABEL: foldv4i64:1142; AVX512VL: # %bb.0:1143; AVX512VL-NEXT: vpmovsxbq {{.*#+}} ymm0 = [55,0,64,56]1144; AVX512VL-NEXT: retq1145;1146; AVX512VLBWDQ-LABEL: foldv4i64:1147; AVX512VLBWDQ: # %bb.0:1148; AVX512VLBWDQ-NEXT: vpmovsxbq {{.*#+}} ymm0 = [55,0,64,56]1149; AVX512VLBWDQ-NEXT: retq1150;1151; AVX512-LABEL: foldv4i64:1152; AVX512: # %bb.0:1153; AVX512-NEXT: vpmovsxbq {{.*#+}} ymm0 = [55,0,64,56]1154; AVX512-NEXT: retq1155;1156; X86-AVX-LABEL: foldv4i64:1157; X86-AVX: # %bb.0:1158; X86-AVX-NEXT: vmovaps {{.*#+}} ymm0 = [55,0,0,0,64,0,56,0]1159; X86-AVX-NEXT: retl1160 %out = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> <i64 256, i64 -1, i64 0, i64 255>, i1 0)1161 ret <4 x i64> %out1162}1163 1164define <4 x i64> @foldv4i64u() nounwind {1165; AVX1-LABEL: foldv4i64u:1166; AVX1: # %bb.0:1167; AVX1-NEXT: vmovaps {{.*#+}} ymm0 = [55,0,64,56]1168; AVX1-NEXT: retq1169;1170; AVX2-LABEL: foldv4i64u:1171; AVX2: # %bb.0:1172; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [55,0,64,56]1173; AVX2-NEXT: retq1174;1175; AVX512VL-LABEL: foldv4i64u:1176; AVX512VL: # %bb.0:1177; AVX512VL-NEXT: vpmovsxbq {{.*#+}} ymm0 = [55,0,64,56]1178; AVX512VL-NEXT: retq1179;1180; AVX512VLBWDQ-LABEL: foldv4i64u:1181; AVX512VLBWDQ: # %bb.0:1182; AVX512VLBWDQ-NEXT: vpmovsxbq {{.*#+}} ymm0 = [55,0,64,56]1183; AVX512VLBWDQ-NEXT: retq1184;1185; AVX512-LABEL: foldv4i64u:1186; AVX512: # %bb.0:1187; AVX512-NEXT: vpmovsxbq {{.*#+}} ymm0 = [55,0,64,56]1188; AVX512-NEXT: retq1189;1190; X86-AVX-LABEL: foldv4i64u:1191; X86-AVX: # %bb.0:1192; X86-AVX-NEXT: vmovaps {{.*#+}} ymm0 = [55,0,0,0,64,0,56,0]1193; X86-AVX-NEXT: retl1194 %out = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> <i64 256, i64 -1, i64 0, i64 255>, i1 -1)1195 ret <4 x i64> %out1196}1197 1198define <8 x i32> @foldv8i32() nounwind {1199; AVX1-LABEL: foldv8i32:1200; AVX1: # %bb.0:1201; AVX1-NEXT: vmovaps {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25]1202; AVX1-NEXT: retq1203;1204; AVX2-LABEL: foldv8i32:1205; AVX2: # %bb.0:1206; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25]1207; AVX2-NEXT: retq1208;1209; AVX512VL-LABEL: foldv8i32:1210; AVX512VL: # %bb.0:1211; AVX512VL-NEXT: vpmovsxbd {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25]1212; AVX512VL-NEXT: retq1213;1214; AVX512VLBWDQ-LABEL: foldv8i32:1215; AVX512VLBWDQ: # %bb.0:1216; AVX512VLBWDQ-NEXT: vpmovsxbd {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25]1217; AVX512VLBWDQ-NEXT: retq1218;1219; AVX512-LABEL: foldv8i32:1220; AVX512: # %bb.0:1221; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25]1222; AVX512-NEXT: retq1223;1224; X86-AVX-LABEL: foldv8i32:1225; X86-AVX: # %bb.0:1226; X86-AVX-NEXT: vmovaps {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25]1227; X86-AVX-NEXT: retl1228 %out = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> <i32 256, i32 -1, i32 0, i32 255, i32 -65536, i32 7, i32 24, i32 88>, i1 0)1229 ret <8 x i32> %out1230}1231 1232define <8 x i32> @foldv8i32u() nounwind {1233; AVX1-LABEL: foldv8i32u:1234; AVX1: # %bb.0:1235; AVX1-NEXT: vmovaps {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25]1236; AVX1-NEXT: retq1237;1238; AVX2-LABEL: foldv8i32u:1239; AVX2: # %bb.0:1240; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25]1241; AVX2-NEXT: retq1242;1243; AVX512VL-LABEL: foldv8i32u:1244; AVX512VL: # %bb.0:1245; AVX512VL-NEXT: vpmovsxbd {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25]1246; AVX512VL-NEXT: retq1247;1248; AVX512VLBWDQ-LABEL: foldv8i32u:1249; AVX512VLBWDQ: # %bb.0:1250; AVX512VLBWDQ-NEXT: vpmovsxbd {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25]1251; AVX512VLBWDQ-NEXT: retq1252;1253; AVX512-LABEL: foldv8i32u:1254; AVX512: # %bb.0:1255; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25]1256; AVX512-NEXT: retq1257;1258; X86-AVX-LABEL: foldv8i32u:1259; X86-AVX: # %bb.0:1260; X86-AVX-NEXT: vmovaps {{.*#+}} ymm0 = [23,0,32,24,0,29,27,25]1261; X86-AVX-NEXT: retl1262 %out = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> <i32 256, i32 -1, i32 0, i32 255, i32 -65536, i32 7, i32 24, i32 88>, i1 -1)1263 ret <8 x i32> %out1264}1265 1266define <16 x i16> @foldv16i16() nounwind {1267; X64-LABEL: foldv16i16:1268; X64: # %bb.0:1269; X64-NEXT: vmovaps {{.*#+}} ymm0 = [7,0,16,8,16,13,11,9,0,8,15,14,13,12,11,10]1270; X64-NEXT: retq1271;1272; X86-AVX-LABEL: foldv16i16:1273; X86-AVX: # %bb.0:1274; X86-AVX-NEXT: vmovaps {{.*#+}} ymm0 = [7,0,16,8,16,13,11,9,0,8,15,14,13,12,11,10]1275; X86-AVX-NEXT: retl1276 %out = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88, i16 -2, i16 254, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32>, i1 0)1277 ret <16 x i16> %out1278}1279 1280define <16 x i16> @foldv16i16u() nounwind {1281; X64-LABEL: foldv16i16u:1282; X64: # %bb.0:1283; X64-NEXT: vmovaps {{.*#+}} ymm0 = [7,0,16,8,16,13,11,9,0,8,15,14,13,12,11,10]1284; X64-NEXT: retq1285;1286; X86-AVX-LABEL: foldv16i16u:1287; X86-AVX: # %bb.0:1288; X86-AVX-NEXT: vmovaps {{.*#+}} ymm0 = [7,0,16,8,16,13,11,9,0,8,15,14,13,12,11,10]1289; X86-AVX-NEXT: retl1290 %out = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> <i16 256, i16 -1, i16 0, i16 255, i16 -65536, i16 7, i16 24, i16 88, i16 -2, i16 254, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32>, i1 -1)1291 ret <16 x i16> %out1292}1293 1294define <32 x i8> @foldv32i8() nounwind {1295; X64-LABEL: foldv32i8:1296; X64: # %bb.0:1297; X64-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2,1,0,8,8,0,0,0,0,0,0,0,0,6,5,5,1]1298; X64-NEXT: retq1299;1300; X86-AVX-LABEL: foldv32i8:1301; X86-AVX: # %bb.0:1302; X86-AVX-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2,1,0,8,8,0,0,0,0,0,0,0,0,6,5,5,1]1303; X86-AVX-NEXT: retl1304 %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 256, i8 -256, i8 -128, i8 -64, i8 -32, i8 -16, i8 -8, i8 -4, i8 -2, i8 -1, i8 3, i8 5, i8 7, i8 127>, i1 0)1305 ret <32 x i8> %out1306}1307 1308define <32 x i8> @foldv32i8u() nounwind {1309; X64-LABEL: foldv32i8u:1310; X64: # %bb.0:1311; X64-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2,1,0,8,8,0,0,0,0,0,0,0,0,6,5,5,1]1312; X64-NEXT: retq1313;1314; X86-AVX-LABEL: foldv32i8u:1315; X86-AVX: # %bb.0:1316; X86-AVX-NEXT: vmovaps {{.*#+}} ymm0 = [8,0,8,0,8,5,3,1,0,0,7,6,5,4,3,2,1,0,8,8,0,0,0,0,0,0,0,0,6,5,5,1]1317; X86-AVX-NEXT: retl1318 %out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> <i8 256, i8 -1, i8 0, i8 255, i8 -65536, i8 7, i8 24, i8 88, i8 -2, i8 254, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 256, i8 -256, i8 -128, i8 -64, i8 -32, i8 -16, i8 -8, i8 -4, i8 -2, i8 -1, i8 3, i8 5, i8 7, i8 127>, i1 -1)1319 ret <32 x i8> %out1320}1321 1322declare <4 x i64> @llvm.ctlz.v4i64(<4 x i64>, i1)1323declare <8 x i32> @llvm.ctlz.v8i32(<8 x i32>, i1)1324declare <16 x i16> @llvm.ctlz.v16i16(<16 x i16>, i1)1325declare <32 x i8> @llvm.ctlz.v32i8(<32 x i8>, i1)1326