658 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512cd | FileCheck %s --check-prefix=AVX512CD3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512cd,+avx512bw | FileCheck %s --check-prefix=AVX512CDBW4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=AVX512BW5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vpopcntdq | FileCheck %s --check-prefix=AVX512VPOPCNTDQ6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bitalg | FileCheck %s --check-prefix=BITALG7 8define <8 x i64> @testv8i64(<8 x i64> %in) nounwind {9; AVX512CD-LABEL: testv8i64:10; AVX512CD: # %bb.0:11; AVX512CD-NEXT: vpternlogd {{.*#+}} zmm1 = -112; AVX512CD-NEXT: vpaddq %zmm1, %zmm0, %zmm113; AVX512CD-NEXT: vpandnq %zmm1, %zmm0, %zmm014; AVX512CD-NEXT: vplzcntq %zmm0, %zmm015; AVX512CD-NEXT: vpbroadcastq {{.*#+}} zmm1 = [64,64,64,64,64,64,64,64]16; AVX512CD-NEXT: vpsubq %zmm0, %zmm1, %zmm017; AVX512CD-NEXT: retq18;19; AVX512CDBW-LABEL: testv8i64:20; AVX512CDBW: # %bb.0:21; AVX512CDBW-NEXT: vpternlogd {{.*#+}} zmm1 = -122; AVX512CDBW-NEXT: vpaddq %zmm1, %zmm0, %zmm123; AVX512CDBW-NEXT: vpandnq %zmm1, %zmm0, %zmm024; AVX512CDBW-NEXT: vplzcntq %zmm0, %zmm025; AVX512CDBW-NEXT: vpbroadcastq {{.*#+}} zmm1 = [64,64,64,64,64,64,64,64]26; AVX512CDBW-NEXT: vpsubq %zmm0, %zmm1, %zmm027; AVX512CDBW-NEXT: retq28;29; AVX512BW-LABEL: testv8i64:30; AVX512BW: # %bb.0:31; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 = -132; AVX512BW-NEXT: vpaddq %zmm1, %zmm0, %zmm133; AVX512BW-NEXT: vpandnq %zmm1, %zmm0, %zmm034; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]35; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm236; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]37; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]38; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm239; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm040; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm041; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm042; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm043; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm144; AVX512BW-NEXT: vpsadbw %zmm1, %zmm0, %zmm045; AVX512BW-NEXT: retq46;47; AVX512VPOPCNTDQ-LABEL: testv8i64:48; AVX512VPOPCNTDQ: # %bb.0:49; AVX512VPOPCNTDQ-NEXT: vpternlogd {{.*#+}} zmm1 = -150; AVX512VPOPCNTDQ-NEXT: vpaddq %zmm1, %zmm0, %zmm151; AVX512VPOPCNTDQ-NEXT: vpandnq %zmm1, %zmm0, %zmm052; AVX512VPOPCNTDQ-NEXT: vpopcntq %zmm0, %zmm053; AVX512VPOPCNTDQ-NEXT: retq54;55; BITALG-LABEL: testv8i64:56; BITALG: # %bb.0:57; BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -158; BITALG-NEXT: vpaddq %zmm1, %zmm0, %zmm159; BITALG-NEXT: vpandnq %zmm1, %zmm0, %zmm060; BITALG-NEXT: vpopcntb %zmm0, %zmm061; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm162; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm063; BITALG-NEXT: retq64 %out = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %in, i1 0)65 ret <8 x i64> %out66}67 68define <8 x i64> @testv8i64u(<8 x i64> %in) nounwind {69; AVX512CD-LABEL: testv8i64u:70; AVX512CD: # %bb.0:71; AVX512CD-NEXT: vpternlogd {{.*#+}} zmm1 = -172; AVX512CD-NEXT: vpaddq %zmm1, %zmm0, %zmm173; AVX512CD-NEXT: vpandnq %zmm1, %zmm0, %zmm074; AVX512CD-NEXT: vplzcntq %zmm0, %zmm075; AVX512CD-NEXT: vpbroadcastq {{.*#+}} zmm1 = [64,64,64,64,64,64,64,64]76; AVX512CD-NEXT: vpsubq %zmm0, %zmm1, %zmm077; AVX512CD-NEXT: retq78;79; AVX512CDBW-LABEL: testv8i64u:80; AVX512CDBW: # %bb.0:81; AVX512CDBW-NEXT: vpternlogd {{.*#+}} zmm1 = -182; AVX512CDBW-NEXT: vpaddq %zmm1, %zmm0, %zmm183; AVX512CDBW-NEXT: vpandnq %zmm1, %zmm0, %zmm084; AVX512CDBW-NEXT: vplzcntq %zmm0, %zmm085; AVX512CDBW-NEXT: vpbroadcastq {{.*#+}} zmm1 = [64,64,64,64,64,64,64,64]86; AVX512CDBW-NEXT: vpsubq %zmm0, %zmm1, %zmm087; AVX512CDBW-NEXT: retq88;89; AVX512BW-LABEL: testv8i64u:90; AVX512BW: # %bb.0:91; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 = -192; AVX512BW-NEXT: vpaddq %zmm1, %zmm0, %zmm193; AVX512BW-NEXT: vpandnq %zmm1, %zmm0, %zmm094; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]95; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm296; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]97; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]98; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm299; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0100; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0101; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0102; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0103; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1104; AVX512BW-NEXT: vpsadbw %zmm1, %zmm0, %zmm0105; AVX512BW-NEXT: retq106;107; AVX512VPOPCNTDQ-LABEL: testv8i64u:108; AVX512VPOPCNTDQ: # %bb.0:109; AVX512VPOPCNTDQ-NEXT: vpternlogd {{.*#+}} zmm1 = -1110; AVX512VPOPCNTDQ-NEXT: vpaddq %zmm1, %zmm0, %zmm1111; AVX512VPOPCNTDQ-NEXT: vpandnq %zmm1, %zmm0, %zmm0112; AVX512VPOPCNTDQ-NEXT: vpopcntq %zmm0, %zmm0113; AVX512VPOPCNTDQ-NEXT: retq114;115; BITALG-LABEL: testv8i64u:116; BITALG: # %bb.0:117; BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1118; BITALG-NEXT: vpaddq %zmm1, %zmm0, %zmm1119; BITALG-NEXT: vpandnq %zmm1, %zmm0, %zmm0120; BITALG-NEXT: vpopcntb %zmm0, %zmm0121; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1122; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0123; BITALG-NEXT: retq124 %out = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %in, i1 -1)125 ret <8 x i64> %out126}127 128define <16 x i32> @testv16i32(<16 x i32> %in) nounwind {129; AVX512CD-LABEL: testv16i32:130; AVX512CD: # %bb.0:131; AVX512CD-NEXT: vpternlogd {{.*#+}} zmm1 = -1132; AVX512CD-NEXT: vpaddd %zmm1, %zmm0, %zmm1133; AVX512CD-NEXT: vpandnd %zmm1, %zmm0, %zmm0134; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0135; AVX512CD-NEXT: vpbroadcastd {{.*#+}} zmm1 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]136; AVX512CD-NEXT: vpsubd %zmm0, %zmm1, %zmm0137; AVX512CD-NEXT: retq138;139; AVX512CDBW-LABEL: testv16i32:140; AVX512CDBW: # %bb.0:141; AVX512CDBW-NEXT: vpternlogd {{.*#+}} zmm1 = -1142; AVX512CDBW-NEXT: vpaddd %zmm1, %zmm0, %zmm1143; AVX512CDBW-NEXT: vpandnd %zmm1, %zmm0, %zmm0144; AVX512CDBW-NEXT: vplzcntd %zmm0, %zmm0145; AVX512CDBW-NEXT: vpbroadcastd {{.*#+}} zmm1 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]146; AVX512CDBW-NEXT: vpsubd %zmm0, %zmm1, %zmm0147; AVX512CDBW-NEXT: retq148;149; AVX512BW-LABEL: testv16i32:150; AVX512BW: # %bb.0:151; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 = -1152; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm1153; AVX512BW-NEXT: vpandnd %zmm1, %zmm0, %zmm0154; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]155; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2156; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]157; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]158; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2159; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0160; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0161; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0162; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0163; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1164; AVX512BW-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]165; AVX512BW-NEXT: vpsadbw %zmm1, %zmm2, %zmm2166; AVX512BW-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]167; AVX512BW-NEXT: vpsadbw %zmm1, %zmm0, %zmm0168; AVX512BW-NEXT: vpackuswb %zmm2, %zmm0, %zmm0169; AVX512BW-NEXT: retq170;171; AVX512VPOPCNTDQ-LABEL: testv16i32:172; AVX512VPOPCNTDQ: # %bb.0:173; AVX512VPOPCNTDQ-NEXT: vpternlogd {{.*#+}} zmm1 = -1174; AVX512VPOPCNTDQ-NEXT: vpaddd %zmm1, %zmm0, %zmm1175; AVX512VPOPCNTDQ-NEXT: vpandnd %zmm1, %zmm0, %zmm0176; AVX512VPOPCNTDQ-NEXT: vpopcntd %zmm0, %zmm0177; AVX512VPOPCNTDQ-NEXT: retq178;179; BITALG-LABEL: testv16i32:180; BITALG: # %bb.0:181; BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1182; BITALG-NEXT: vpaddd %zmm1, %zmm0, %zmm1183; BITALG-NEXT: vpandnd %zmm1, %zmm0, %zmm0184; BITALG-NEXT: vpopcntb %zmm0, %zmm0185; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1186; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]187; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2188; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]189; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0190; BITALG-NEXT: vpackuswb %zmm2, %zmm0, %zmm0191; BITALG-NEXT: retq192 %out = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %in, i1 0)193 ret <16 x i32> %out194}195 196define <16 x i32> @testv16i32u(<16 x i32> %in) nounwind {197; AVX512CD-LABEL: testv16i32u:198; AVX512CD: # %bb.0:199; AVX512CD-NEXT: vpternlogd {{.*#+}} zmm1 = -1200; AVX512CD-NEXT: vpaddd %zmm1, %zmm0, %zmm1201; AVX512CD-NEXT: vpandnd %zmm1, %zmm0, %zmm0202; AVX512CD-NEXT: vplzcntd %zmm0, %zmm0203; AVX512CD-NEXT: vpbroadcastd {{.*#+}} zmm1 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]204; AVX512CD-NEXT: vpsubd %zmm0, %zmm1, %zmm0205; AVX512CD-NEXT: retq206;207; AVX512CDBW-LABEL: testv16i32u:208; AVX512CDBW: # %bb.0:209; AVX512CDBW-NEXT: vpternlogd {{.*#+}} zmm1 = -1210; AVX512CDBW-NEXT: vpaddd %zmm1, %zmm0, %zmm1211; AVX512CDBW-NEXT: vpandnd %zmm1, %zmm0, %zmm0212; AVX512CDBW-NEXT: vplzcntd %zmm0, %zmm0213; AVX512CDBW-NEXT: vpbroadcastd {{.*#+}} zmm1 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]214; AVX512CDBW-NEXT: vpsubd %zmm0, %zmm1, %zmm0215; AVX512CDBW-NEXT: retq216;217; AVX512BW-LABEL: testv16i32u:218; AVX512BW: # %bb.0:219; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 = -1220; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm1221; AVX512BW-NEXT: vpandnd %zmm1, %zmm0, %zmm0222; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]223; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2224; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]225; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]226; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2227; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0228; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0229; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0230; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0231; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1232; AVX512BW-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]233; AVX512BW-NEXT: vpsadbw %zmm1, %zmm2, %zmm2234; AVX512BW-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]235; AVX512BW-NEXT: vpsadbw %zmm1, %zmm0, %zmm0236; AVX512BW-NEXT: vpackuswb %zmm2, %zmm0, %zmm0237; AVX512BW-NEXT: retq238;239; AVX512VPOPCNTDQ-LABEL: testv16i32u:240; AVX512VPOPCNTDQ: # %bb.0:241; AVX512VPOPCNTDQ-NEXT: vpternlogd {{.*#+}} zmm1 = -1242; AVX512VPOPCNTDQ-NEXT: vpaddd %zmm1, %zmm0, %zmm1243; AVX512VPOPCNTDQ-NEXT: vpandnd %zmm1, %zmm0, %zmm0244; AVX512VPOPCNTDQ-NEXT: vpopcntd %zmm0, %zmm0245; AVX512VPOPCNTDQ-NEXT: retq246;247; BITALG-LABEL: testv16i32u:248; BITALG: # %bb.0:249; BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1250; BITALG-NEXT: vpaddd %zmm1, %zmm0, %zmm1251; BITALG-NEXT: vpandnd %zmm1, %zmm0, %zmm0252; BITALG-NEXT: vpopcntb %zmm0, %zmm0253; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1254; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]255; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2256; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]257; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0258; BITALG-NEXT: vpackuswb %zmm2, %zmm0, %zmm0259; BITALG-NEXT: retq260 %out = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %in, i1 -1)261 ret <16 x i32> %out262}263 264define <32 x i16> @testv32i16(<32 x i16> %in) nounwind {265; AVX512CD-LABEL: testv32i16:266; AVX512CD: # %bb.0:267; AVX512CD-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1268; AVX512CD-NEXT: vpaddw %ymm1, %ymm0, %ymm2269; AVX512CD-NEXT: vpandn %ymm2, %ymm0, %ymm2270; AVX512CD-NEXT: vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]271; AVX512CD-NEXT: vpand %ymm3, %ymm2, %ymm4272; AVX512CD-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]273; AVX512CD-NEXT: # ymm5 = mem[0,1,0,1]274; AVX512CD-NEXT: vpshufb %ymm4, %ymm5, %ymm4275; AVX512CD-NEXT: vpsrlw $4, %ymm2, %ymm2276; AVX512CD-NEXT: vpand %ymm3, %ymm2, %ymm2277; AVX512CD-NEXT: vpshufb %ymm2, %ymm5, %ymm2278; AVX512CD-NEXT: vpaddb %ymm4, %ymm2, %ymm2279; AVX512CD-NEXT: vpsllw $8, %ymm2, %ymm4280; AVX512CD-NEXT: vpaddb %ymm2, %ymm4, %ymm2281; AVX512CD-NEXT: vpsrlw $8, %ymm2, %ymm2282; AVX512CD-NEXT: vextracti64x4 $1, %zmm0, %ymm0283; AVX512CD-NEXT: vpaddw %ymm1, %ymm0, %ymm1284; AVX512CD-NEXT: vpandn %ymm1, %ymm0, %ymm0285; AVX512CD-NEXT: vpand %ymm3, %ymm0, %ymm1286; AVX512CD-NEXT: vpshufb %ymm1, %ymm5, %ymm1287; AVX512CD-NEXT: vpsrlw $4, %ymm0, %ymm0288; AVX512CD-NEXT: vpand %ymm3, %ymm0, %ymm0289; AVX512CD-NEXT: vpshufb %ymm0, %ymm5, %ymm0290; AVX512CD-NEXT: vpaddb %ymm1, %ymm0, %ymm0291; AVX512CD-NEXT: vpsllw $8, %ymm0, %ymm1292; AVX512CD-NEXT: vpaddb %ymm0, %ymm1, %ymm0293; AVX512CD-NEXT: vpsrlw $8, %ymm0, %ymm0294; AVX512CD-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0295; AVX512CD-NEXT: retq296;297; AVX512CDBW-LABEL: testv32i16:298; AVX512CDBW: # %bb.0:299; AVX512CDBW-NEXT: vpternlogd {{.*#+}} zmm1 = -1300; AVX512CDBW-NEXT: vpaddw %zmm1, %zmm0, %zmm1301; AVX512CDBW-NEXT: vpandnq %zmm1, %zmm0, %zmm0302; AVX512CDBW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]303; AVX512CDBW-NEXT: vpandq %zmm1, %zmm0, %zmm2304; AVX512CDBW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]305; AVX512CDBW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]306; AVX512CDBW-NEXT: vpshufb %zmm2, %zmm3, %zmm2307; AVX512CDBW-NEXT: vpsrlw $4, %zmm0, %zmm0308; AVX512CDBW-NEXT: vpandq %zmm1, %zmm0, %zmm0309; AVX512CDBW-NEXT: vpshufb %zmm0, %zmm3, %zmm0310; AVX512CDBW-NEXT: vpaddb %zmm2, %zmm0, %zmm0311; AVX512CDBW-NEXT: vpsllw $8, %zmm0, %zmm1312; AVX512CDBW-NEXT: vpaddb %zmm0, %zmm1, %zmm0313; AVX512CDBW-NEXT: vpsrlw $8, %zmm0, %zmm0314; AVX512CDBW-NEXT: retq315;316; AVX512BW-LABEL: testv32i16:317; AVX512BW: # %bb.0:318; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 = -1319; AVX512BW-NEXT: vpaddw %zmm1, %zmm0, %zmm1320; AVX512BW-NEXT: vpandnq %zmm1, %zmm0, %zmm0321; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]322; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2323; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]324; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]325; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2326; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0327; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0328; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0329; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0330; AVX512BW-NEXT: vpsllw $8, %zmm0, %zmm1331; AVX512BW-NEXT: vpaddb %zmm0, %zmm1, %zmm0332; AVX512BW-NEXT: vpsrlw $8, %zmm0, %zmm0333; AVX512BW-NEXT: retq334;335; AVX512VPOPCNTDQ-LABEL: testv32i16:336; AVX512VPOPCNTDQ: # %bb.0:337; AVX512VPOPCNTDQ-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1338; AVX512VPOPCNTDQ-NEXT: vpaddw %ymm1, %ymm0, %ymm2339; AVX512VPOPCNTDQ-NEXT: vpandn %ymm2, %ymm0, %ymm2340; AVX512VPOPCNTDQ-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero341; AVX512VPOPCNTDQ-NEXT: vpopcntd %zmm2, %zmm2342; AVX512VPOPCNTDQ-NEXT: vpmovdw %zmm2, %ymm2343; AVX512VPOPCNTDQ-NEXT: vextracti64x4 $1, %zmm0, %ymm0344; AVX512VPOPCNTDQ-NEXT: vpaddw %ymm1, %ymm0, %ymm1345; AVX512VPOPCNTDQ-NEXT: vpandn %ymm1, %ymm0, %ymm0346; AVX512VPOPCNTDQ-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero347; AVX512VPOPCNTDQ-NEXT: vpopcntd %zmm0, %zmm0348; AVX512VPOPCNTDQ-NEXT: vpmovdw %zmm0, %ymm0349; AVX512VPOPCNTDQ-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0350; AVX512VPOPCNTDQ-NEXT: retq351;352; BITALG-LABEL: testv32i16:353; BITALG: # %bb.0:354; BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1355; BITALG-NEXT: vpaddw %zmm1, %zmm0, %zmm1356; BITALG-NEXT: vpandnq %zmm1, %zmm0, %zmm0357; BITALG-NEXT: vpopcntw %zmm0, %zmm0358; BITALG-NEXT: retq359 %out = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %in, i1 0)360 ret <32 x i16> %out361}362 363define <32 x i16> @testv32i16u(<32 x i16> %in) nounwind {364; AVX512CD-LABEL: testv32i16u:365; AVX512CD: # %bb.0:366; AVX512CD-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1367; AVX512CD-NEXT: vpaddw %ymm1, %ymm0, %ymm2368; AVX512CD-NEXT: vpandn %ymm2, %ymm0, %ymm2369; AVX512CD-NEXT: vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]370; AVX512CD-NEXT: vpand %ymm3, %ymm2, %ymm4371; AVX512CD-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]372; AVX512CD-NEXT: # ymm5 = mem[0,1,0,1]373; AVX512CD-NEXT: vpshufb %ymm4, %ymm5, %ymm4374; AVX512CD-NEXT: vpsrlw $4, %ymm2, %ymm2375; AVX512CD-NEXT: vpand %ymm3, %ymm2, %ymm2376; AVX512CD-NEXT: vpshufb %ymm2, %ymm5, %ymm2377; AVX512CD-NEXT: vpaddb %ymm4, %ymm2, %ymm2378; AVX512CD-NEXT: vpsllw $8, %ymm2, %ymm4379; AVX512CD-NEXT: vpaddb %ymm2, %ymm4, %ymm2380; AVX512CD-NEXT: vpsrlw $8, %ymm2, %ymm2381; AVX512CD-NEXT: vextracti64x4 $1, %zmm0, %ymm0382; AVX512CD-NEXT: vpaddw %ymm1, %ymm0, %ymm1383; AVX512CD-NEXT: vpandn %ymm1, %ymm0, %ymm0384; AVX512CD-NEXT: vpand %ymm3, %ymm0, %ymm1385; AVX512CD-NEXT: vpshufb %ymm1, %ymm5, %ymm1386; AVX512CD-NEXT: vpsrlw $4, %ymm0, %ymm0387; AVX512CD-NEXT: vpand %ymm3, %ymm0, %ymm0388; AVX512CD-NEXT: vpshufb %ymm0, %ymm5, %ymm0389; AVX512CD-NEXT: vpaddb %ymm1, %ymm0, %ymm0390; AVX512CD-NEXT: vpsllw $8, %ymm0, %ymm1391; AVX512CD-NEXT: vpaddb %ymm0, %ymm1, %ymm0392; AVX512CD-NEXT: vpsrlw $8, %ymm0, %ymm0393; AVX512CD-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0394; AVX512CD-NEXT: retq395;396; AVX512CDBW-LABEL: testv32i16u:397; AVX512CDBW: # %bb.0:398; AVX512CDBW-NEXT: vpternlogd {{.*#+}} zmm1 = -1399; AVX512CDBW-NEXT: vpaddw %zmm1, %zmm0, %zmm1400; AVX512CDBW-NEXT: vpandnq %zmm1, %zmm0, %zmm0401; AVX512CDBW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]402; AVX512CDBW-NEXT: vpandq %zmm1, %zmm0, %zmm2403; AVX512CDBW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]404; AVX512CDBW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]405; AVX512CDBW-NEXT: vpshufb %zmm2, %zmm3, %zmm2406; AVX512CDBW-NEXT: vpsrlw $4, %zmm0, %zmm0407; AVX512CDBW-NEXT: vpandq %zmm1, %zmm0, %zmm0408; AVX512CDBW-NEXT: vpshufb %zmm0, %zmm3, %zmm0409; AVX512CDBW-NEXT: vpaddb %zmm2, %zmm0, %zmm0410; AVX512CDBW-NEXT: vpsllw $8, %zmm0, %zmm1411; AVX512CDBW-NEXT: vpaddb %zmm0, %zmm1, %zmm0412; AVX512CDBW-NEXT: vpsrlw $8, %zmm0, %zmm0413; AVX512CDBW-NEXT: retq414;415; AVX512BW-LABEL: testv32i16u:416; AVX512BW: # %bb.0:417; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 = -1418; AVX512BW-NEXT: vpaddw %zmm1, %zmm0, %zmm1419; AVX512BW-NEXT: vpandnq %zmm1, %zmm0, %zmm0420; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]421; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2422; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]423; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]424; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2425; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0426; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0427; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0428; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0429; AVX512BW-NEXT: vpsllw $8, %zmm0, %zmm1430; AVX512BW-NEXT: vpaddb %zmm0, %zmm1, %zmm0431; AVX512BW-NEXT: vpsrlw $8, %zmm0, %zmm0432; AVX512BW-NEXT: retq433;434; AVX512VPOPCNTDQ-LABEL: testv32i16u:435; AVX512VPOPCNTDQ: # %bb.0:436; AVX512VPOPCNTDQ-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1437; AVX512VPOPCNTDQ-NEXT: vpaddw %ymm1, %ymm0, %ymm2438; AVX512VPOPCNTDQ-NEXT: vpandn %ymm2, %ymm0, %ymm2439; AVX512VPOPCNTDQ-NEXT: vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero440; AVX512VPOPCNTDQ-NEXT: vpopcntd %zmm2, %zmm2441; AVX512VPOPCNTDQ-NEXT: vpmovdw %zmm2, %ymm2442; AVX512VPOPCNTDQ-NEXT: vextracti64x4 $1, %zmm0, %ymm0443; AVX512VPOPCNTDQ-NEXT: vpaddw %ymm1, %ymm0, %ymm1444; AVX512VPOPCNTDQ-NEXT: vpandn %ymm1, %ymm0, %ymm0445; AVX512VPOPCNTDQ-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero446; AVX512VPOPCNTDQ-NEXT: vpopcntd %zmm0, %zmm0447; AVX512VPOPCNTDQ-NEXT: vpmovdw %zmm0, %ymm0448; AVX512VPOPCNTDQ-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0449; AVX512VPOPCNTDQ-NEXT: retq450;451; BITALG-LABEL: testv32i16u:452; BITALG: # %bb.0:453; BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1454; BITALG-NEXT: vpaddw %zmm1, %zmm0, %zmm1455; BITALG-NEXT: vpandnq %zmm1, %zmm0, %zmm0456; BITALG-NEXT: vpopcntw %zmm0, %zmm0457; BITALG-NEXT: retq458 %out = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %in, i1 -1)459 ret <32 x i16> %out460}461 462define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {463; AVX512CD-LABEL: testv64i8:464; AVX512CD: # %bb.0:465; AVX512CD-NEXT: vextracti64x4 $1, %zmm0, %ymm1466; AVX512CD-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2467; AVX512CD-NEXT: vpaddb %ymm2, %ymm1, %ymm3468; AVX512CD-NEXT: vpandn %ymm3, %ymm1, %ymm1469; AVX512CD-NEXT: vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]470; AVX512CD-NEXT: vpand %ymm3, %ymm1, %ymm4471; AVX512CD-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]472; AVX512CD-NEXT: # ymm5 = mem[0,1,0,1]473; AVX512CD-NEXT: vpshufb %ymm4, %ymm5, %ymm4474; AVX512CD-NEXT: vpsrlw $4, %ymm1, %ymm1475; AVX512CD-NEXT: vpand %ymm3, %ymm1, %ymm1476; AVX512CD-NEXT: vpshufb %ymm1, %ymm5, %ymm1477; AVX512CD-NEXT: vpaddb %ymm4, %ymm1, %ymm1478; AVX512CD-NEXT: vpaddb %ymm2, %ymm0, %ymm2479; AVX512CD-NEXT: vpandn %ymm2, %ymm0, %ymm0480; AVX512CD-NEXT: vpand %ymm3, %ymm0, %ymm2481; AVX512CD-NEXT: vpshufb %ymm2, %ymm5, %ymm2482; AVX512CD-NEXT: vpsrlw $4, %ymm0, %ymm0483; AVX512CD-NEXT: vpand %ymm3, %ymm0, %ymm0484; AVX512CD-NEXT: vpshufb %ymm0, %ymm5, %ymm0485; AVX512CD-NEXT: vpaddb %ymm2, %ymm0, %ymm0486; AVX512CD-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0487; AVX512CD-NEXT: retq488;489; AVX512CDBW-LABEL: testv64i8:490; AVX512CDBW: # %bb.0:491; AVX512CDBW-NEXT: vpternlogd {{.*#+}} zmm1 = -1492; AVX512CDBW-NEXT: vpaddb %zmm1, %zmm0, %zmm1493; AVX512CDBW-NEXT: vpandnq %zmm1, %zmm0, %zmm0494; AVX512CDBW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]495; AVX512CDBW-NEXT: vpandq %zmm1, %zmm0, %zmm2496; AVX512CDBW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]497; AVX512CDBW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]498; AVX512CDBW-NEXT: vpshufb %zmm2, %zmm3, %zmm2499; AVX512CDBW-NEXT: vpsrlw $4, %zmm0, %zmm0500; AVX512CDBW-NEXT: vpandq %zmm1, %zmm0, %zmm0501; AVX512CDBW-NEXT: vpshufb %zmm0, %zmm3, %zmm0502; AVX512CDBW-NEXT: vpaddb %zmm2, %zmm0, %zmm0503; AVX512CDBW-NEXT: retq504;505; AVX512BW-LABEL: testv64i8:506; AVX512BW: # %bb.0:507; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 = -1508; AVX512BW-NEXT: vpaddb %zmm1, %zmm0, %zmm1509; AVX512BW-NEXT: vpandnq %zmm1, %zmm0, %zmm0510; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]511; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2512; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]513; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]514; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2515; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0516; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0517; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0518; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0519; AVX512BW-NEXT: retq520;521; AVX512VPOPCNTDQ-LABEL: testv64i8:522; AVX512VPOPCNTDQ: # %bb.0:523; AVX512VPOPCNTDQ-NEXT: vextracti64x4 $1, %zmm0, %ymm1524; AVX512VPOPCNTDQ-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2525; AVX512VPOPCNTDQ-NEXT: vpaddb %ymm2, %ymm1, %ymm3526; AVX512VPOPCNTDQ-NEXT: vpandn %ymm3, %ymm1, %ymm1527; AVX512VPOPCNTDQ-NEXT: vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]528; AVX512VPOPCNTDQ-NEXT: vpand %ymm3, %ymm1, %ymm4529; AVX512VPOPCNTDQ-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]530; AVX512VPOPCNTDQ-NEXT: # ymm5 = mem[0,1,0,1]531; AVX512VPOPCNTDQ-NEXT: vpshufb %ymm4, %ymm5, %ymm4532; AVX512VPOPCNTDQ-NEXT: vpsrlw $4, %ymm1, %ymm1533; AVX512VPOPCNTDQ-NEXT: vpand %ymm3, %ymm1, %ymm1534; AVX512VPOPCNTDQ-NEXT: vpshufb %ymm1, %ymm5, %ymm1535; AVX512VPOPCNTDQ-NEXT: vpaddb %ymm4, %ymm1, %ymm1536; AVX512VPOPCNTDQ-NEXT: vpaddb %ymm2, %ymm0, %ymm2537; AVX512VPOPCNTDQ-NEXT: vpandn %ymm2, %ymm0, %ymm0538; AVX512VPOPCNTDQ-NEXT: vpand %ymm3, %ymm0, %ymm2539; AVX512VPOPCNTDQ-NEXT: vpshufb %ymm2, %ymm5, %ymm2540; AVX512VPOPCNTDQ-NEXT: vpsrlw $4, %ymm0, %ymm0541; AVX512VPOPCNTDQ-NEXT: vpand %ymm3, %ymm0, %ymm0542; AVX512VPOPCNTDQ-NEXT: vpshufb %ymm0, %ymm5, %ymm0543; AVX512VPOPCNTDQ-NEXT: vpaddb %ymm2, %ymm0, %ymm0544; AVX512VPOPCNTDQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0545; AVX512VPOPCNTDQ-NEXT: retq546;547; BITALG-LABEL: testv64i8:548; BITALG: # %bb.0:549; BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1550; BITALG-NEXT: vpaddb %zmm1, %zmm0, %zmm1551; BITALG-NEXT: vpandnq %zmm1, %zmm0, %zmm0552; BITALG-NEXT: vpopcntb %zmm0, %zmm0553; BITALG-NEXT: retq554 %out = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %in, i1 0)555 ret <64 x i8> %out556}557 558define <64 x i8> @testv64i8u(<64 x i8> %in) nounwind {559; AVX512CD-LABEL: testv64i8u:560; AVX512CD: # %bb.0:561; AVX512CD-NEXT: vextracti64x4 $1, %zmm0, %ymm1562; AVX512CD-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2563; AVX512CD-NEXT: vpaddb %ymm2, %ymm1, %ymm3564; AVX512CD-NEXT: vpandn %ymm3, %ymm1, %ymm1565; AVX512CD-NEXT: vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]566; AVX512CD-NEXT: vpand %ymm3, %ymm1, %ymm4567; AVX512CD-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]568; AVX512CD-NEXT: # ymm5 = mem[0,1,0,1]569; AVX512CD-NEXT: vpshufb %ymm4, %ymm5, %ymm4570; AVX512CD-NEXT: vpsrlw $4, %ymm1, %ymm1571; AVX512CD-NEXT: vpand %ymm3, %ymm1, %ymm1572; AVX512CD-NEXT: vpshufb %ymm1, %ymm5, %ymm1573; AVX512CD-NEXT: vpaddb %ymm4, %ymm1, %ymm1574; AVX512CD-NEXT: vpaddb %ymm2, %ymm0, %ymm2575; AVX512CD-NEXT: vpandn %ymm2, %ymm0, %ymm0576; AVX512CD-NEXT: vpand %ymm3, %ymm0, %ymm2577; AVX512CD-NEXT: vpshufb %ymm2, %ymm5, %ymm2578; AVX512CD-NEXT: vpsrlw $4, %ymm0, %ymm0579; AVX512CD-NEXT: vpand %ymm3, %ymm0, %ymm0580; AVX512CD-NEXT: vpshufb %ymm0, %ymm5, %ymm0581; AVX512CD-NEXT: vpaddb %ymm2, %ymm0, %ymm0582; AVX512CD-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0583; AVX512CD-NEXT: retq584;585; AVX512CDBW-LABEL: testv64i8u:586; AVX512CDBW: # %bb.0:587; AVX512CDBW-NEXT: vpternlogd {{.*#+}} zmm1 = -1588; AVX512CDBW-NEXT: vpaddb %zmm1, %zmm0, %zmm1589; AVX512CDBW-NEXT: vpandnq %zmm1, %zmm0, %zmm0590; AVX512CDBW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]591; AVX512CDBW-NEXT: vpandq %zmm1, %zmm0, %zmm2592; AVX512CDBW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]593; AVX512CDBW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]594; AVX512CDBW-NEXT: vpshufb %zmm2, %zmm3, %zmm2595; AVX512CDBW-NEXT: vpsrlw $4, %zmm0, %zmm0596; AVX512CDBW-NEXT: vpandq %zmm1, %zmm0, %zmm0597; AVX512CDBW-NEXT: vpshufb %zmm0, %zmm3, %zmm0598; AVX512CDBW-NEXT: vpaddb %zmm2, %zmm0, %zmm0599; AVX512CDBW-NEXT: retq600;601; AVX512BW-LABEL: testv64i8u:602; AVX512BW: # %bb.0:603; AVX512BW-NEXT: vpternlogd {{.*#+}} zmm1 = -1604; AVX512BW-NEXT: vpaddb %zmm1, %zmm0, %zmm1605; AVX512BW-NEXT: vpandnq %zmm1, %zmm0, %zmm0606; AVX512BW-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]607; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm2608; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]609; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]610; AVX512BW-NEXT: vpshufb %zmm2, %zmm3, %zmm2611; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0612; AVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0613; AVX512BW-NEXT: vpshufb %zmm0, %zmm3, %zmm0614; AVX512BW-NEXT: vpaddb %zmm2, %zmm0, %zmm0615; AVX512BW-NEXT: retq616;617; AVX512VPOPCNTDQ-LABEL: testv64i8u:618; AVX512VPOPCNTDQ: # %bb.0:619; AVX512VPOPCNTDQ-NEXT: vextracti64x4 $1, %zmm0, %ymm1620; AVX512VPOPCNTDQ-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2621; AVX512VPOPCNTDQ-NEXT: vpaddb %ymm2, %ymm1, %ymm3622; AVX512VPOPCNTDQ-NEXT: vpandn %ymm3, %ymm1, %ymm1623; AVX512VPOPCNTDQ-NEXT: vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]624; AVX512VPOPCNTDQ-NEXT: vpand %ymm3, %ymm1, %ymm4625; AVX512VPOPCNTDQ-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]626; AVX512VPOPCNTDQ-NEXT: # ymm5 = mem[0,1,0,1]627; AVX512VPOPCNTDQ-NEXT: vpshufb %ymm4, %ymm5, %ymm4628; AVX512VPOPCNTDQ-NEXT: vpsrlw $4, %ymm1, %ymm1629; AVX512VPOPCNTDQ-NEXT: vpand %ymm3, %ymm1, %ymm1630; AVX512VPOPCNTDQ-NEXT: vpshufb %ymm1, %ymm5, %ymm1631; AVX512VPOPCNTDQ-NEXT: vpaddb %ymm4, %ymm1, %ymm1632; AVX512VPOPCNTDQ-NEXT: vpaddb %ymm2, %ymm0, %ymm2633; AVX512VPOPCNTDQ-NEXT: vpandn %ymm2, %ymm0, %ymm0634; AVX512VPOPCNTDQ-NEXT: vpand %ymm3, %ymm0, %ymm2635; AVX512VPOPCNTDQ-NEXT: vpshufb %ymm2, %ymm5, %ymm2636; AVX512VPOPCNTDQ-NEXT: vpsrlw $4, %ymm0, %ymm0637; AVX512VPOPCNTDQ-NEXT: vpand %ymm3, %ymm0, %ymm0638; AVX512VPOPCNTDQ-NEXT: vpshufb %ymm0, %ymm5, %ymm0639; AVX512VPOPCNTDQ-NEXT: vpaddb %ymm2, %ymm0, %ymm0640; AVX512VPOPCNTDQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0641; AVX512VPOPCNTDQ-NEXT: retq642;643; BITALG-LABEL: testv64i8u:644; BITALG: # %bb.0:645; BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1646; BITALG-NEXT: vpaddb %zmm1, %zmm0, %zmm1647; BITALG-NEXT: vpandnq %zmm1, %zmm0, %zmm0648; BITALG-NEXT: vpopcntb %zmm0, %zmm0649; BITALG-NEXT: retq650 %out = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %in, i1 -1)651 ret <64 x i8> %out652}653 654declare <8 x i64> @llvm.cttz.v8i64(<8 x i64>, i1)655declare <16 x i32> @llvm.cttz.v16i32(<16 x i32>, i1)656declare <32 x i16> @llvm.cttz.v32i16(<32 x i16>, i1)657declare <64 x i8> @llvm.cttz.v64i8(<64 x i8>, i1)658