755 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX5123; RUN: llc < %s -mtriple=x86_64-- -mcpu=knl | FileCheck %s --check-prefixes=AVX512F4 5; Tests for BMI1/BMI2/TBM style bit manipulations that could potentially stay on the predicate registers6 7; ANDNOT - Logical and not8 9define <8 x i64> @andnot_v8i64(<8 x i64> %a0, <8 x i64> %a1, i8 %a2) {10; AVX512-LABEL: andnot_v8i64:11; AVX512: # %bb.0:12; AVX512-NEXT: notb %dil13; AVX512-NEXT: kmovd %edi, %k114; AVX512-NEXT: vpcmpgtq %zmm1, %zmm0, %k1 {%k1}15; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm116; AVX512-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}17; AVX512-NEXT: vmovdqa64 %zmm1, %zmm018; AVX512-NEXT: retq19;20; AVX512F-LABEL: andnot_v8i64:21; AVX512F: # %bb.0:22; AVX512F-NEXT: notb %dil23; AVX512F-NEXT: kmovw %edi, %k124; AVX512F-NEXT: vpcmpgtq %zmm1, %zmm0, %k1 {%k1}25; AVX512F-NEXT: vpaddq %zmm1, %zmm0, %zmm126; AVX512F-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}27; AVX512F-NEXT: vmovdqa64 %zmm1, %zmm028; AVX512F-NEXT: retq29 %cmp = icmp sgt <8 x i64> %a0, %a130 %mask = bitcast <8 x i1> %cmp to i831 %not = xor i8 %a2, -132 %andnot = and i8 %mask, %not33 %sel = bitcast i8 %andnot to <8 x i1>34 %add = add <8 x i64> %a0, %a135 %res = select <8 x i1> %sel, <8 x i64> %a0, <8 x i64> %add36 ret <8 x i64> %res37}38 39; BEXTR - Bit field extract (register)40 41define <32 x i16> @bextr_reg_v32i16(<32 x i16> %a0, <32 x i16> %a1, i32 %idx, i32 %len) {42; AVX512-LABEL: bextr_reg_v32i16:43; AVX512: # %bb.0:44; AVX512-NEXT: vpcmpgtw %zmm1, %zmm0, %k045; AVX512-NEXT: kmovd %k0, %eax46; AVX512-NEXT: shrxl %edi, %eax, %eax47; AVX512-NEXT: bzhil %esi, %eax, %eax48; AVX512-NEXT: kmovd %eax, %k149; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm150; AVX512-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1}51; AVX512-NEXT: vmovdqa64 %zmm1, %zmm052; AVX512-NEXT: retq53;54; AVX512F-LABEL: bextr_reg_v32i16:55; AVX512F: # %bb.0:56; AVX512F-NEXT: vpcmpgtw %ymm1, %ymm0, %ymm257; AVX512F-NEXT: vpmovsxwd %ymm2, %zmm258; AVX512F-NEXT: vptestmd %zmm2, %zmm2, %k059; AVX512F-NEXT: kmovw %k0, %eax60; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm261; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm362; AVX512F-NEXT: vpcmpgtw %ymm2, %ymm3, %ymm463; AVX512F-NEXT: vpmovsxwd %ymm4, %zmm464; AVX512F-NEXT: vptestmd %zmm4, %zmm4, %k065; AVX512F-NEXT: kmovw %k0, %ecx66; AVX512F-NEXT: shll $16, %ecx67; AVX512F-NEXT: orl %eax, %ecx68; AVX512F-NEXT: shrxl %edi, %ecx, %eax69; AVX512F-NEXT: bzhil %esi, %eax, %eax70; AVX512F-NEXT: kmovw %eax, %k171; AVX512F-NEXT: shrl $16, %eax72; AVX512F-NEXT: kmovw %eax, %k273; AVX512F-NEXT: vpaddw %ymm2, %ymm3, %ymm274; AVX512F-NEXT: vpaddw %ymm1, %ymm0, %ymm175; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm176; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k1} {z} = -177; AVX512F-NEXT: vpmovdw %zmm2, %ymm278; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k2} {z} = -179; AVX512F-NEXT: vpmovdw %zmm3, %ymm380; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm281; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm2 & (zmm0 ^ zmm1))82; AVX512F-NEXT: retq83 %cmp = icmp sgt <32 x i16> %a0, %a184 %mask = bitcast <32 x i1> %cmp to i3285 %shift = lshr i32 %mask, %idx86 %bit = shl i32 1, %len87 %msk = sub i32 %bit, 188 %bextr = and i32 %shift, %msk89 %sel = bitcast i32 %bextr to <32 x i1>90 %add = add <32 x i16> %a0, %a191 %res = select <32 x i1> %sel, <32 x i16> %a0, <32 x i16> %add92 ret <32 x i16> %res93}94 95; BEXTR - Bit field extract (immediate)96 97define <32 x i16> @bextr_imm_v32i16(<32 x i16> %a0, <32 x i16> %a1) {98; AVX512-LABEL: bextr_imm_v32i16:99; AVX512: # %bb.0:100; AVX512-NEXT: vpcmpgtw %zmm1, %zmm0, %k0101; AVX512-NEXT: kmovd %k0, %eax102; AVX512-NEXT: shrl $2, %eax103; AVX512-NEXT: andl $7, %eax104; AVX512-NEXT: kmovd %eax, %k1105; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm1106; AVX512-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1}107; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0108; AVX512-NEXT: retq109;110; AVX512F-LABEL: bextr_imm_v32i16:111; AVX512F: # %bb.0:112; AVX512F-NEXT: vpcmpgtw %ymm1, %ymm0, %ymm2113; AVX512F-NEXT: vpmovsxwd %ymm2, %zmm2114; AVX512F-NEXT: vptestmd %zmm2, %zmm2, %k0115; AVX512F-NEXT: kmovw %k0, %eax116; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2117; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3118; AVX512F-NEXT: vpaddw %ymm2, %ymm3, %ymm2119; AVX512F-NEXT: shrl $2, %eax120; AVX512F-NEXT: andl $7, %eax121; AVX512F-NEXT: kmovw %eax, %k1122; AVX512F-NEXT: vpaddw %ymm1, %ymm0, %ymm1123; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1124; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1125; AVX512F-NEXT: vpmovdw %zmm2, %ymm2126; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm2 & (zmm0 ^ zmm1))127; AVX512F-NEXT: retq128 %cmp = icmp sgt <32 x i16> %a0, %a1129 %mask = bitcast <32 x i1> %cmp to i32130 %shift = lshr i32 %mask, 2131 %bextr = and i32 %shift, 7132 %sel = bitcast i32 %bextr to <32 x i1>133 %add = add <32 x i16> %a0, %a1134 %res = select <32 x i1> %sel, <32 x i16> %a0, <32 x i16> %add135 ret <32 x i16> %res136}137 138; BLSI - Extract lowest set isolated bit (x & -x)139 140define <64 x i8> @blsi_v64i8(<64 x i8> %a0, <64 x i8> %a1) {141; AVX512-LABEL: blsi_v64i8:142; AVX512: # %bb.0:143; AVX512-NEXT: vpcmpgtb %zmm1, %zmm0, %k0144; AVX512-NEXT: kmovq %k0, %rax145; AVX512-NEXT: blsiq %rax, %rax146; AVX512-NEXT: kmovq %rax, %k1147; AVX512-NEXT: vpaddb %zmm1, %zmm0, %zmm1148; AVX512-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1}149; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0150; AVX512-NEXT: retq151;152; AVX512F-LABEL: blsi_v64i8:153; AVX512F: # %bb.0:154; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2155; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3156; AVX512F-NEXT: vpcmpgtb %ymm2, %ymm3, %ymm4157; AVX512F-NEXT: vpcmpgtb %ymm1, %ymm0, %ymm5158; AVX512F-NEXT: vpmovmskb %ymm5, %eax159; AVX512F-NEXT: vpmovmskb %ymm4, %ecx160; AVX512F-NEXT: shlq $32, %rcx161; AVX512F-NEXT: orq %rax, %rcx162; AVX512F-NEXT: blsiq %rcx, %rax163; AVX512F-NEXT: movq %rax, %rcx164; AVX512F-NEXT: movl %eax, %edx165; AVX512F-NEXT: kmovw %eax, %k1166; AVX512F-NEXT: shrq $32, %rax167; AVX512F-NEXT: shrq $48, %rcx168; AVX512F-NEXT: shrl $16, %edx169; AVX512F-NEXT: kmovw %edx, %k2170; AVX512F-NEXT: kmovw %ecx, %k3171; AVX512F-NEXT: kmovw %eax, %k4172; AVX512F-NEXT: vpaddb %ymm2, %ymm3, %ymm2173; AVX512F-NEXT: vpaddb %ymm1, %ymm0, %ymm1174; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1175; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k4} {z} = -1176; AVX512F-NEXT: vpmovdb %zmm2, %xmm2177; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k3} {z} = -1178; AVX512F-NEXT: vpmovdb %zmm3, %xmm3179; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2180; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1181; AVX512F-NEXT: vpmovdb %zmm3, %xmm3182; AVX512F-NEXT: vpternlogd {{.*#+}} zmm4 {%k2} {z} = -1183; AVX512F-NEXT: vpmovdb %zmm4, %xmm4184; AVX512F-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3185; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2186; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm2 & (zmm0 ^ zmm1))187; AVX512F-NEXT: retq188 %cmp = icmp sgt <64 x i8> %a0, %a1189 %mask = bitcast <64 x i1> %cmp to i64190 %neg = sub i64 0, %mask191 %blsi = and i64 %mask, %neg192 %sel = bitcast i64 %blsi to <64 x i1>193 %add = add <64 x i8> %a0, %a1194 %res = select <64 x i1> %sel, <64 x i8> %a0, <64 x i8> %add195 ret <64 x i8> %res196}197 198; BLSMSK - Get mask up to lowest set bit (x ^ (x - 1))199 200define <16 x float> @blsmsk_v16f32(<16 x float> %a0, <16 x float> %a1) {201; AVX512-LABEL: blsmsk_v16f32:202; AVX512: # %bb.0:203; AVX512-NEXT: vcmpltps %zmm0, %zmm1, %k0204; AVX512-NEXT: kmovd %k0, %eax205; AVX512-NEXT: blsmskl %eax, %eax206; AVX512-NEXT: kmovd %eax, %k1207; AVX512-NEXT: vaddps %zmm1, %zmm0, %zmm1208; AVX512-NEXT: vmovaps %zmm0, %zmm1 {%k1}209; AVX512-NEXT: vmovaps %zmm1, %zmm0210; AVX512-NEXT: retq211;212; AVX512F-LABEL: blsmsk_v16f32:213; AVX512F: # %bb.0:214; AVX512F-NEXT: vcmpltps %zmm0, %zmm1, %k0215; AVX512F-NEXT: kmovw %k0, %eax216; AVX512F-NEXT: blsmskl %eax, %eax217; AVX512F-NEXT: kmovw %eax, %k1218; AVX512F-NEXT: vaddps %zmm1, %zmm0, %zmm1219; AVX512F-NEXT: vmovaps %zmm0, %zmm1 {%k1}220; AVX512F-NEXT: vmovaps %zmm1, %zmm0221; AVX512F-NEXT: retq222 %cmp = fcmp ogt <16 x float> %a0, %a1223 %mask = bitcast <16 x i1> %cmp to i16224 %dec = sub i16 %mask, 1225 %blsmsk = xor i16 %mask, %dec226 %sel = bitcast i16 %blsmsk to <16 x i1>227 %add = fadd <16 x float> %a0, %a1228 %res = select <16 x i1> %sel, <16 x float> %a0, <16 x float> %add229 ret <16 x float> %res230}231 232; BLSR - Reset lowest set bit (x & (x - 1))233 234define <8 x double> @blsr_v8f64(<8 x double> %a0, <8 x double> %a1) {235; AVX512-LABEL: blsr_v8f64:236; AVX512: # %bb.0:237; AVX512-NEXT: vcmpltpd %zmm0, %zmm1, %k0238; AVX512-NEXT: kmovd %k0, %eax239; AVX512-NEXT: leal -1(%rax), %ecx240; AVX512-NEXT: andb %al, %cl241; AVX512-NEXT: kmovd %ecx, %k1242; AVX512-NEXT: vaddpd %zmm1, %zmm0, %zmm1243; AVX512-NEXT: vmovapd %zmm0, %zmm1 {%k1}244; AVX512-NEXT: vmovapd %zmm1, %zmm0245; AVX512-NEXT: retq246;247; AVX512F-LABEL: blsr_v8f64:248; AVX512F: # %bb.0:249; AVX512F-NEXT: vcmpltpd %zmm0, %zmm1, %k0250; AVX512F-NEXT: kmovw %k0, %eax251; AVX512F-NEXT: leal -1(%rax), %ecx252; AVX512F-NEXT: andb %al, %cl253; AVX512F-NEXT: kmovw %ecx, %k1254; AVX512F-NEXT: vaddpd %zmm1, %zmm0, %zmm1255; AVX512F-NEXT: vmovapd %zmm0, %zmm1 {%k1}256; AVX512F-NEXT: vmovapd %zmm1, %zmm0257; AVX512F-NEXT: retq258 %cmp = fcmp ogt <8 x double> %a0, %a1259 %mask = bitcast <8 x i1> %cmp to i8260 %dec = sub i8 %mask, 1261 %blsr = and i8 %mask, %dec262 %sel = bitcast i8 %blsr to <8 x i1>263 %add = fadd <8 x double> %a0, %a1264 %res = select <8 x i1> %sel, <8 x double> %a0, <8 x double> %add265 ret <8 x double> %res266}267 268; BZHI - Zero high bits starting from specified index (x & ((1 << idx) - 1))269 270define <16 x i32> @bzhi_v16i32(<16 x i32> %a0, <16 x i32> %a1, i16 %idx) {271; AVX512-LABEL: bzhi_v16i32:272; AVX512: # %bb.0:273; AVX512-NEXT: movl $-1, %eax274; AVX512-NEXT: bzhil %edi, %eax, %eax275; AVX512-NEXT: kmovd %eax, %k1276; AVX512-NEXT: vpcmpnleud %zmm1, %zmm0, %k1 {%k1}277; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm1278; AVX512-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}279; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0280; AVX512-NEXT: retq281;282; AVX512F-LABEL: bzhi_v16i32:283; AVX512F: # %bb.0:284; AVX512F-NEXT: movl $-1, %eax285; AVX512F-NEXT: bzhil %edi, %eax, %eax286; AVX512F-NEXT: kmovw %eax, %k1287; AVX512F-NEXT: vpcmpnleud %zmm1, %zmm0, %k1 {%k1}288; AVX512F-NEXT: vpaddd %zmm1, %zmm0, %zmm1289; AVX512F-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}290; AVX512F-NEXT: vmovdqa64 %zmm1, %zmm0291; AVX512F-NEXT: retq292 %cmp = icmp ugt <16 x i32> %a0, %a1293 %mask = bitcast <16 x i1> %cmp to i16294 %bit = shl i16 1, %idx295 %msk = sub i16 %bit, 1296 %bzhi = and i16 %mask, %msk297 %sel = bitcast i16 %bzhi to <16 x i1>298 %add = add <16 x i32> %a0, %a1299 %res = select <16 x i1> %sel, <16 x i32> %a0, <16 x i32> %add300 ret <16 x i32> %res301}302 303; BLCFILL - Fill from lowest clear bit (x & (x + 1))304 305define <32 x i16> @blcfill_v32i16(<32 x i16> %a0, <32 x i16> %a1) {306; AVX512-LABEL: blcfill_v32i16:307; AVX512: # %bb.0:308; AVX512-NEXT: vpcmpgtw %zmm1, %zmm0, %k0309; AVX512-NEXT: kmovd %k0, %eax310; AVX512-NEXT: leal 1(%rax), %ecx311; AVX512-NEXT: andl %eax, %ecx312; AVX512-NEXT: kmovd %ecx, %k1313; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm1314; AVX512-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1}315; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0316; AVX512-NEXT: retq317;318; AVX512F-LABEL: blcfill_v32i16:319; AVX512F: # %bb.0:320; AVX512F-NEXT: vpcmpgtw %ymm1, %ymm0, %ymm2321; AVX512F-NEXT: vpmovsxwd %ymm2, %zmm2322; AVX512F-NEXT: vptestmd %zmm2, %zmm2, %k0323; AVX512F-NEXT: kmovw %k0, %eax324; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2325; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3326; AVX512F-NEXT: vpcmpgtw %ymm2, %ymm3, %ymm4327; AVX512F-NEXT: vpmovsxwd %ymm4, %zmm4328; AVX512F-NEXT: vptestmd %zmm4, %zmm4, %k0329; AVX512F-NEXT: kmovw %k0, %ecx330; AVX512F-NEXT: shll $16, %ecx331; AVX512F-NEXT: leal (%rax,%rcx), %edx332; AVX512F-NEXT: addl $1, %edx333; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax334; AVX512F-NEXT: orl %ecx, %eax335; AVX512F-NEXT: andl %eax, %edx336; AVX512F-NEXT: kmovw %edx, %k1337; AVX512F-NEXT: shrl $16, %edx338; AVX512F-NEXT: kmovw %edx, %k2339; AVX512F-NEXT: vpaddw %ymm2, %ymm3, %ymm2340; AVX512F-NEXT: vpaddw %ymm1, %ymm0, %ymm1341; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1342; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1343; AVX512F-NEXT: vpmovdw %zmm2, %ymm2344; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k2} {z} = -1345; AVX512F-NEXT: vpmovdw %zmm3, %ymm3346; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2347; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm2 & (zmm0 ^ zmm1))348; AVX512F-NEXT: retq349 %cmp = icmp sgt <32 x i16> %a0, %a1350 %mask = bitcast <32 x i1> %cmp to i32351 %inc = add i32 %mask, 1352 %blcfill = and i32 %mask, %inc353 %sel = bitcast i32 %blcfill to <32 x i1>354 %add = add <32 x i16> %a0, %a1355 %res = select <32 x i1> %sel, <32 x i16> %a0, <32 x i16> %add356 ret <32 x i16> %res357}358 359; BLCI - Isolate lowest clear bit (x | ~(x + 1))360 361define <64 x i8> @blci_v64i8(<64 x i8> %a0, <64 x i8> %a1) {362; AVX512-LABEL: blci_v64i8:363; AVX512: # %bb.0:364; AVX512-NEXT: vpcmpgtb %zmm1, %zmm0, %k0365; AVX512-NEXT: kmovq %k0, %rax366; AVX512-NEXT: leaq 1(%rax), %rcx367; AVX512-NEXT: notq %rcx368; AVX512-NEXT: orq %rax, %rcx369; AVX512-NEXT: kmovq %rcx, %k1370; AVX512-NEXT: vpaddb %zmm1, %zmm0, %zmm1371; AVX512-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1}372; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0373; AVX512-NEXT: retq374;375; AVX512F-LABEL: blci_v64i8:376; AVX512F: # %bb.0:377; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2378; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3379; AVX512F-NEXT: vpcmpgtb %ymm2, %ymm3, %ymm4380; AVX512F-NEXT: vpcmpgtb %ymm1, %ymm0, %ymm5381; AVX512F-NEXT: vpmovmskb %ymm5, %eax382; AVX512F-NEXT: vpmovmskb %ymm4, %ecx383; AVX512F-NEXT: shlq $32, %rcx384; AVX512F-NEXT: leaq (%rax,%rcx), %rdx385; AVX512F-NEXT: addq %rcx, %rax386; AVX512F-NEXT: addq $1, %rax387; AVX512F-NEXT: notq %rax388; AVX512F-NEXT: orq %rdx, %rax389; AVX512F-NEXT: movq %rax, %rcx390; AVX512F-NEXT: movl %eax, %edx391; AVX512F-NEXT: kmovw %eax, %k1392; AVX512F-NEXT: shrq $32, %rax393; AVX512F-NEXT: shrq $48, %rcx394; AVX512F-NEXT: shrl $16, %edx395; AVX512F-NEXT: kmovw %edx, %k2396; AVX512F-NEXT: kmovw %ecx, %k3397; AVX512F-NEXT: kmovw %eax, %k4398; AVX512F-NEXT: vpaddb %ymm2, %ymm3, %ymm2399; AVX512F-NEXT: vpaddb %ymm1, %ymm0, %ymm1400; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1401; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k4} {z} = -1402; AVX512F-NEXT: vpmovdb %zmm2, %xmm2403; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k3} {z} = -1404; AVX512F-NEXT: vpmovdb %zmm3, %xmm3405; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2406; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1407; AVX512F-NEXT: vpmovdb %zmm3, %xmm3408; AVX512F-NEXT: vpternlogd {{.*#+}} zmm4 {%k2} {z} = -1409; AVX512F-NEXT: vpmovdb %zmm4, %xmm4410; AVX512F-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3411; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2412; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm2 & (zmm0 ^ zmm1))413; AVX512F-NEXT: retq414 %cmp = icmp sgt <64 x i8> %a0, %a1415 %mask = bitcast <64 x i1> %cmp to i64416 %inc = add i64 %mask, 1417 %not = xor i64 %inc, -1418 %blci = or i64 %mask, %not419 %sel = bitcast i64 %blci to <64 x i1>420 %add = add <64 x i8> %a0, %a1421 %res = select <64 x i1> %sel, <64 x i8> %a0, <64 x i8> %add422 ret <64 x i8> %res423}424 425; BLCIC - Isolate lowest clear bit and complement (~x & (x + 1))426 427define <8 x i64> @blcic_v8i64(<8 x i64> %a0, <8 x i64> %a1) {428; AVX512-LABEL: blcic_v8i64:429; AVX512: # %bb.0:430; AVX512-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0431; AVX512-NEXT: kmovd %k0, %eax432; AVX512-NEXT: movl %eax, %ecx433; AVX512-NEXT: notb %cl434; AVX512-NEXT: incb %al435; AVX512-NEXT: andb %cl, %al436; AVX512-NEXT: kmovd %eax, %k1437; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm1438; AVX512-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}439; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0440; AVX512-NEXT: retq441;442; AVX512F-LABEL: blcic_v8i64:443; AVX512F: # %bb.0:444; AVX512F-NEXT: vpcmpnltuq %zmm1, %zmm0, %k0445; AVX512F-NEXT: kmovw %k0, %eax446; AVX512F-NEXT: movl %eax, %ecx447; AVX512F-NEXT: notb %cl448; AVX512F-NEXT: addb $1, %al449; AVX512F-NEXT: andb %cl, %al450; AVX512F-NEXT: kmovw %eax, %k1451; AVX512F-NEXT: vpaddq %zmm1, %zmm0, %zmm1452; AVX512F-NEXT: vmovdqa64 %zmm0, %zmm1 {%k1}453; AVX512F-NEXT: vmovdqa64 %zmm1, %zmm0454; AVX512F-NEXT: retq455 %cmp = icmp uge <8 x i64> %a0, %a1456 %mask = bitcast <8 x i1> %cmp to i8457 %not = xor i8 %mask, -1458 %inc = add i8 %mask, 1459 %blcic = and i8 %not, %inc460 %sel = bitcast i8 %blcic to <8 x i1>461 %add = add <8 x i64> %a0, %a1462 %res = select <8 x i1> %sel, <8 x i64> %a0, <8 x i64> %add463 ret <8 x i64> %res464}465 466; BLCMSK - Mask from lowest clear bit (x ^ (x + 1))467 468define <32 x i16> @blcmsk_v32i16(<32 x i16> %a0, <32 x i16> %a1) {469; AVX512-LABEL: blcmsk_v32i16:470; AVX512: # %bb.0:471; AVX512-NEXT: vpcmpgtw %zmm1, %zmm0, %k0472; AVX512-NEXT: kmovd %k0, %eax473; AVX512-NEXT: leal 1(%rax), %ecx474; AVX512-NEXT: xorl %eax, %ecx475; AVX512-NEXT: kmovd %ecx, %k1476; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm1477; AVX512-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1}478; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0479; AVX512-NEXT: retq480;481; AVX512F-LABEL: blcmsk_v32i16:482; AVX512F: # %bb.0:483; AVX512F-NEXT: vpcmpgtw %ymm1, %ymm0, %ymm2484; AVX512F-NEXT: vpmovsxwd %ymm2, %zmm2485; AVX512F-NEXT: vptestmd %zmm2, %zmm2, %k0486; AVX512F-NEXT: kmovw %k0, %eax487; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2488; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3489; AVX512F-NEXT: vpcmpgtw %ymm2, %ymm3, %ymm4490; AVX512F-NEXT: vpmovsxwd %ymm4, %zmm4491; AVX512F-NEXT: vptestmd %zmm4, %zmm4, %k0492; AVX512F-NEXT: kmovw %k0, %ecx493; AVX512F-NEXT: shll $16, %ecx494; AVX512F-NEXT: leal (%rax,%rcx), %edx495; AVX512F-NEXT: addl $1, %edx496; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax497; AVX512F-NEXT: orl %ecx, %eax498; AVX512F-NEXT: xorl %eax, %edx499; AVX512F-NEXT: kmovw %edx, %k1500; AVX512F-NEXT: shrl $16, %edx501; AVX512F-NEXT: kmovw %edx, %k2502; AVX512F-NEXT: vpaddw %ymm2, %ymm3, %ymm2503; AVX512F-NEXT: vpaddw %ymm1, %ymm0, %ymm1504; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1505; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1506; AVX512F-NEXT: vpmovdw %zmm2, %ymm2507; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k2} {z} = -1508; AVX512F-NEXT: vpmovdw %zmm3, %ymm3509; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2510; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm2 & (zmm0 ^ zmm1))511; AVX512F-NEXT: retq512 %cmp = icmp sgt <32 x i16> %a0, %a1513 %mask = bitcast <32 x i1> %cmp to i32514 %inc = add i32 %mask, 1515 %blcmsk = xor i32 %mask, %inc516 %sel = bitcast i32 %blcmsk to <32 x i1>517 %add = add <32 x i16> %a0, %a1518 %res = select <32 x i1> %sel, <32 x i16> %a0, <32 x i16> %add519 ret <32 x i16> %res520}521 522; BLCS - Set lowest clear bit (x | (x + 1))523 524define <16 x float> @blcs_v16f32(<16 x float> %a0, <16 x float> %a1) {525; AVX512-LABEL: blcs_v16f32:526; AVX512: # %bb.0:527; AVX512-NEXT: vcmpltps %zmm0, %zmm1, %k0528; AVX512-NEXT: kmovd %k0, %eax529; AVX512-NEXT: leal 1(%rax), %ecx530; AVX512-NEXT: orl %eax, %ecx531; AVX512-NEXT: kmovd %ecx, %k1532; AVX512-NEXT: vaddps %zmm1, %zmm0, %zmm1533; AVX512-NEXT: vmovaps %zmm0, %zmm1 {%k1}534; AVX512-NEXT: vmovaps %zmm1, %zmm0535; AVX512-NEXT: retq536;537; AVX512F-LABEL: blcs_v16f32:538; AVX512F: # %bb.0:539; AVX512F-NEXT: vcmpltps %zmm0, %zmm1, %k0540; AVX512F-NEXT: kmovw %k0, %eax541; AVX512F-NEXT: leal 1(%rax), %ecx542; AVX512F-NEXT: orl %eax, %ecx543; AVX512F-NEXT: kmovw %ecx, %k1544; AVX512F-NEXT: vaddps %zmm1, %zmm0, %zmm1545; AVX512F-NEXT: vmovaps %zmm0, %zmm1 {%k1}546; AVX512F-NEXT: vmovaps %zmm1, %zmm0547; AVX512F-NEXT: retq548 %cmp = fcmp ogt <16 x float> %a0, %a1549 %mask = bitcast <16 x i1> %cmp to i16550 %inc = add i16 %mask, 1551 %blcs = or i16 %mask, %inc552 %sel = bitcast i16 %blcs to <16 x i1>553 %add = fadd <16 x float> %a0, %a1554 %res = select <16 x i1> %sel, <16 x float> %a0, <16 x float> %add555 ret <16 x float> %res556}557 558; BLSFILL - Fill from lowest set bit (x | (x - 1))559 560define <16 x i32> @blcs_v16i32(<16 x i32> %a0, <16 x i32> %a1) {561; AVX512-LABEL: blcs_v16i32:562; AVX512: # %bb.0:563; AVX512-NEXT: vpcmpled %zmm1, %zmm0, %k0564; AVX512-NEXT: kmovd %k0, %eax565; AVX512-NEXT: leal -1(%rax), %ecx566; AVX512-NEXT: orl %eax, %ecx567; AVX512-NEXT: kmovd %ecx, %k1568; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm1569; AVX512-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}570; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0571; AVX512-NEXT: retq572;573; AVX512F-LABEL: blcs_v16i32:574; AVX512F: # %bb.0:575; AVX512F-NEXT: vpcmpled %zmm1, %zmm0, %k0576; AVX512F-NEXT: kmovw %k0, %eax577; AVX512F-NEXT: leal -1(%rax), %ecx578; AVX512F-NEXT: orl %eax, %ecx579; AVX512F-NEXT: kmovw %ecx, %k1580; AVX512F-NEXT: vpaddd %zmm1, %zmm0, %zmm1581; AVX512F-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}582; AVX512F-NEXT: vmovdqa64 %zmm1, %zmm0583; AVX512F-NEXT: retq584 %cmp = icmp sle <16 x i32> %a0, %a1585 %mask = bitcast <16 x i1> %cmp to i16586 %dec = sub i16 %mask, 1587 %blsfill = or i16 %mask, %dec588 %sel = bitcast i16 %blsfill to <16 x i1>589 %add = add <16 x i32> %a0, %a1590 %res = select <16 x i1> %sel, <16 x i32> %a0, <16 x i32> %add591 ret <16 x i32> %res592}593 594; BLSIC - Isolate lowest set bit and complement (~x | (x - 1))595 596define <8 x double> @blsic_v8f64(<8 x double> %a0, <8 x double> %a1) {597; AVX512-LABEL: blsic_v8f64:598; AVX512: # %bb.0:599; AVX512-NEXT: vcmpltpd %zmm0, %zmm1, %k0600; AVX512-NEXT: kmovd %k0, %eax601; AVX512-NEXT: movl %eax, %ecx602; AVX512-NEXT: notb %cl603; AVX512-NEXT: decb %al604; AVX512-NEXT: orb %cl, %al605; AVX512-NEXT: kmovd %eax, %k1606; AVX512-NEXT: vaddpd %zmm1, %zmm0, %zmm1607; AVX512-NEXT: vmovapd %zmm0, %zmm1 {%k1}608; AVX512-NEXT: vmovapd %zmm1, %zmm0609; AVX512-NEXT: retq610;611; AVX512F-LABEL: blsic_v8f64:612; AVX512F: # %bb.0:613; AVX512F-NEXT: vcmpltpd %zmm0, %zmm1, %k0614; AVX512F-NEXT: kmovw %k0, %eax615; AVX512F-NEXT: movl %eax, %ecx616; AVX512F-NEXT: notb %cl617; AVX512F-NEXT: addb $-1, %al618; AVX512F-NEXT: orb %cl, %al619; AVX512F-NEXT: kmovw %eax, %k1620; AVX512F-NEXT: vaddpd %zmm1, %zmm0, %zmm1621; AVX512F-NEXT: vmovapd %zmm0, %zmm1 {%k1}622; AVX512F-NEXT: vmovapd %zmm1, %zmm0623; AVX512F-NEXT: retq624 %cmp = fcmp ogt <8 x double> %a0, %a1625 %mask = bitcast <8 x i1> %cmp to i8626 %not = xor i8 %mask, -1627 %dec = sub i8 %mask, 1628 %blsic = or i8 %not, %dec629 %sel = bitcast i8 %blsic to <8 x i1>630 %add = fadd <8 x double> %a0, %a1631 %res = select <8 x i1> %sel, <8 x double> %a0, <8 x double> %add632 ret <8 x double> %res633}634 635; T1MSKC - Inverse mask from trailing ones (~x | (x + 1))636 637define <32 x i16> @t1mskc_v32i16(<32 x i16> %a0, <32 x i16> %a1) {638; AVX512-LABEL: t1mskc_v32i16:639; AVX512: # %bb.0:640; AVX512-NEXT: vpcmpgtw %zmm1, %zmm0, %k0641; AVX512-NEXT: kmovd %k0, %eax642; AVX512-NEXT: leal 1(%rax), %ecx643; AVX512-NEXT: orl %eax, %ecx644; AVX512-NEXT: kmovd %ecx, %k1645; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm1646; AVX512-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1}647; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0648; AVX512-NEXT: retq649;650; AVX512F-LABEL: t1mskc_v32i16:651; AVX512F: # %bb.0:652; AVX512F-NEXT: vpcmpgtw %ymm1, %ymm0, %ymm2653; AVX512F-NEXT: vpmovsxwd %ymm2, %zmm2654; AVX512F-NEXT: vptestmd %zmm2, %zmm2, %k0655; AVX512F-NEXT: kmovw %k0, %eax656; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2657; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3658; AVX512F-NEXT: vpcmpgtw %ymm2, %ymm3, %ymm4659; AVX512F-NEXT: vpmovsxwd %ymm4, %zmm4660; AVX512F-NEXT: vptestmd %zmm4, %zmm4, %k0661; AVX512F-NEXT: kmovw %k0, %ecx662; AVX512F-NEXT: shll $16, %ecx663; AVX512F-NEXT: leal (%rax,%rcx), %edx664; AVX512F-NEXT: addl $1, %edx665; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax666; AVX512F-NEXT: orl %ecx, %eax667; AVX512F-NEXT: orl %eax, %edx668; AVX512F-NEXT: kmovw %edx, %k1669; AVX512F-NEXT: shrl $16, %edx670; AVX512F-NEXT: kmovw %edx, %k2671; AVX512F-NEXT: vpaddw %ymm2, %ymm3, %ymm2672; AVX512F-NEXT: vpaddw %ymm1, %ymm0, %ymm1673; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1674; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1675; AVX512F-NEXT: vpmovdw %zmm2, %ymm2676; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k2} {z} = -1677; AVX512F-NEXT: vpmovdw %zmm3, %ymm3678; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2679; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm2 & (zmm0 ^ zmm1))680; AVX512F-NEXT: retq681 %cmp = icmp sgt <32 x i16> %a0, %a1682 %mask = bitcast <32 x i1> %cmp to i32683 %not = xor i32 %mask, -1684 %inc = add i32 %mask, 1685 %t1mskc = or i32 %mask, %inc686 %sel = bitcast i32 %t1mskc to <32 x i1>687 %add = add <32 x i16> %a0, %a1688 %res = select <32 x i1> %sel, <32 x i16> %a0, <32 x i16> %add689 ret <32 x i16> %res690}691 692; TZMSK - Mask from trailing zeros (~x & (x - 1))693 694define <64 x i8> @tzmsk_v64i8(<64 x i8> %a0, <64 x i8> %a1) {695; AVX512-LABEL: tzmsk_v64i8:696; AVX512: # %bb.0:697; AVX512-NEXT: vpcmpgtb %zmm1, %zmm0, %k0698; AVX512-NEXT: kmovq %k0, %rax699; AVX512-NEXT: leaq -1(%rax), %rcx700; AVX512-NEXT: andnq %rcx, %rax, %rax701; AVX512-NEXT: kmovq %rax, %k1702; AVX512-NEXT: vpaddb %zmm1, %zmm0, %zmm1703; AVX512-NEXT: vmovdqu8 %zmm0, %zmm1 {%k1}704; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0705; AVX512-NEXT: retq706;707; AVX512F-LABEL: tzmsk_v64i8:708; AVX512F: # %bb.0:709; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2710; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3711; AVX512F-NEXT: vpcmpgtb %ymm2, %ymm3, %ymm4712; AVX512F-NEXT: vpcmpgtb %ymm1, %ymm0, %ymm5713; AVX512F-NEXT: vpmovmskb %ymm5, %eax714; AVX512F-NEXT: vpmovmskb %ymm4, %ecx715; AVX512F-NEXT: shlq $32, %rcx716; AVX512F-NEXT: leaq (%rax,%rcx), %rdx717; AVX512F-NEXT: addq %rcx, %rax718; AVX512F-NEXT: addq $-1, %rax719; AVX512F-NEXT: andnq %rax, %rdx, %rax720; AVX512F-NEXT: movq %rax, %rcx721; AVX512F-NEXT: movl %eax, %edx722; AVX512F-NEXT: kmovw %eax, %k1723; AVX512F-NEXT: shrq $32, %rax724; AVX512F-NEXT: shrq $48, %rcx725; AVX512F-NEXT: shrl $16, %edx726; AVX512F-NEXT: kmovw %edx, %k2727; AVX512F-NEXT: kmovw %ecx, %k3728; AVX512F-NEXT: kmovw %eax, %k4729; AVX512F-NEXT: vpaddb %ymm2, %ymm3, %ymm2730; AVX512F-NEXT: vpaddb %ymm1, %ymm0, %ymm1731; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1732; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k4} {z} = -1733; AVX512F-NEXT: vpmovdb %zmm2, %xmm2734; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k3} {z} = -1735; AVX512F-NEXT: vpmovdb %zmm3, %xmm3736; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2737; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1738; AVX512F-NEXT: vpmovdb %zmm3, %xmm3739; AVX512F-NEXT: vpternlogd {{.*#+}} zmm4 {%k2} {z} = -1740; AVX512F-NEXT: vpmovdb %zmm4, %xmm4741; AVX512F-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3742; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2743; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm2 & (zmm0 ^ zmm1))744; AVX512F-NEXT: retq745 %cmp = icmp sgt <64 x i8> %a0, %a1746 %mask = bitcast <64 x i1> %cmp to i64747 %not = xor i64 %mask, -1748 %dec = sub i64 %mask, 1749 %tzmsk = and i64 %not, %dec750 %sel = bitcast i64 %tzmsk to <64 x i1>751 %add = add <64 x i8> %a0, %a1752 %res = select <64 x i1> %sel, <64 x i8> %a0, <64 x i8> %add753 ret <64 x i8> %res754}755