230 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=AVX512,AVX512DQ4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512DQBW6 7declare <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr>, i32, <16 x i1>, <16 x float>)8declare <16 x float> @llvm.masked.expandload.v16f32(ptr, <16 x i1>, <16 x float>)9declare <8 x float> @llvm.masked.expandload.v8f32(ptr, <8 x i1>, <8 x float>)10declare <16 x i32> @llvm.masked.expandload.v16i32(ptr, <16 x i1>, <16 x i32>)11 12; Test case 1: Direct v8i1 all-ones mask (should use kxnorb on AVX512DQ)13define <8 x float> @mask_v8i1_allones(ptr %ptr) {14; AVX512F-LABEL: mask_v8i1_allones:15; AVX512F: # %bb.0:16; AVX512F-NEXT: movw $255, %ax17; AVX512F-NEXT: kmovw %eax, %k118; AVX512F-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}19; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm020; AVX512F-NEXT: retq21;22; AVX512DQ-LABEL: mask_v8i1_allones:23; AVX512DQ: # %bb.0:24; AVX512DQ-NEXT: kxnorb %k0, %k0, %k125; AVX512DQ-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}26; AVX512DQ-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm027; AVX512DQ-NEXT: retq28;29; AVX512BW-LABEL: mask_v8i1_allones:30; AVX512BW: # %bb.0:31; AVX512BW-NEXT: movw $255, %ax32; AVX512BW-NEXT: kmovd %eax, %k133; AVX512BW-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}34; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm035; AVX512BW-NEXT: retq36;37; AVX512DQBW-LABEL: mask_v8i1_allones:38; AVX512DQBW: # %bb.0:39; AVX512DQBW-NEXT: kxnorb %k0, %k0, %k140; AVX512DQBW-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}41; AVX512DQBW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm042; AVX512DQBW-NEXT: retq43 %res = call <8 x float> @llvm.masked.expandload.v8f32(ptr %ptr, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x float> zeroinitializer)44 ret <8 x float> %res45}46 47; Test case 2: v16i1 with lower 8 bits set via bitconvert (should use kxnorb on AVX512DQ)48define <16 x float> @mask_v16i1_lower8(ptr %ptr) {49; AVX512F-LABEL: mask_v16i1_lower8:50; AVX512F: # %bb.0:51; AVX512F-NEXT: movw $255, %ax52; AVX512F-NEXT: kmovw %eax, %k153; AVX512F-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}54; AVX512F-NEXT: retq55;56; AVX512DQ-LABEL: mask_v16i1_lower8:57; AVX512DQ: # %bb.0:58; AVX512DQ-NEXT: kxnorb %k0, %k0, %k159; AVX512DQ-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}60; AVX512DQ-NEXT: retq61;62; AVX512BW-LABEL: mask_v16i1_lower8:63; AVX512BW: # %bb.0:64; AVX512BW-NEXT: movw $255, %ax65; AVX512BW-NEXT: kmovd %eax, %k166; AVX512BW-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}67; AVX512BW-NEXT: retq68;69; AVX512DQBW-LABEL: mask_v16i1_lower8:70; AVX512DQBW: # %bb.0:71; AVX512DQBW-NEXT: kxnorb %k0, %k0, %k172; AVX512DQBW-NEXT: vexpandps (%rdi), %zmm0 {%k1} {z}73; AVX512DQBW-NEXT: retq74 %res = call <16 x float> @llvm.masked.expandload.v16f32(ptr %ptr, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>, <16 x float> zeroinitializer)75 ret <16 x float> %res76}77 78; Test case 3: v16i1 with all bits set (should use kxnorw on all targets)79define <16 x float> @gather_all(ptr %base, <16 x i32> %ind, i16 %mask) {80; AVX512-LABEL: gather_all:81; AVX512: # %bb.0:82; AVX512-NEXT: kxnorw %k0, %k0, %k183; AVX512-NEXT: vxorps %xmm1, %xmm1, %xmm184; AVX512-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1}85; AVX512-NEXT: vmovaps %zmm1, %zmm086; AVX512-NEXT: retq87 %broadcast.splatinsert = insertelement <16 x ptr> poison, ptr %base, i32 088 %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> poison, <16 x i32> zeroinitializer89 %sext_ind = sext <16 x i32> %ind to <16 x i64>90 %gep.random = getelementptr float, <16 x ptr> %broadcast.splat, <16 x i64> %sext_ind91 %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> poison)92 ret <16 x float> %res93}94 95; Test case 4: v8i1 with lower 8 bits set in gather (should use kxnorb on AVX512DQ targets)96define <16 x float> @gather_lower(ptr %base, <16 x i32> %ind, i16 %mask) {97; AVX512F-LABEL: gather_lower:98; AVX512F: # %bb.0:99; AVX512F-NEXT: vxorps %xmm1, %xmm1, %xmm1100; AVX512F-NEXT: movw $255, %ax101; AVX512F-NEXT: kmovw %eax, %k1102; AVX512F-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1}103; AVX512F-NEXT: vmovaps %zmm1, %zmm0104; AVX512F-NEXT: retq105;106; AVX512DQ-LABEL: gather_lower:107; AVX512DQ: # %bb.0:108; AVX512DQ-NEXT: vxorps %xmm1, %xmm1, %xmm1109; AVX512DQ-NEXT: kxnorb %k0, %k0, %k1110; AVX512DQ-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1}111; AVX512DQ-NEXT: vmovaps %zmm1, %zmm0112; AVX512DQ-NEXT: retq113;114; AVX512BW-LABEL: gather_lower:115; AVX512BW: # %bb.0:116; AVX512BW-NEXT: vxorps %xmm1, %xmm1, %xmm1117; AVX512BW-NEXT: movw $255, %ax118; AVX512BW-NEXT: kmovd %eax, %k1119; AVX512BW-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1}120; AVX512BW-NEXT: vmovaps %zmm1, %zmm0121; AVX512BW-NEXT: retq122;123; AVX512DQBW-LABEL: gather_lower:124; AVX512DQBW: # %bb.0:125; AVX512DQBW-NEXT: vxorps %xmm1, %xmm1, %xmm1126; AVX512DQBW-NEXT: kxnorb %k0, %k0, %k1127; AVX512DQBW-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1}128; AVX512DQBW-NEXT: vmovaps %zmm1, %zmm0129; AVX512DQBW-NEXT: retq130 %broadcast.splatinsert = insertelement <16 x ptr> poison, ptr %base, i32 0131 %broadcast.splat = shufflevector <16 x ptr> %broadcast.splatinsert, <16 x ptr> poison, <16 x i32> zeroinitializer132 %sext_ind = sext <16 x i32> %ind to <16 x i64>133 %gep.random = getelementptr float, <16 x ptr> %broadcast.splat, <16 x i64> %sext_ind134 %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>, <16 x float> poison)135 ret <16 x float> %res136}137 138; Test case 5: v32i1 mask via bitconvert combined with dynamic condition.139; Ensures lower 16 lanes force the KSET1W path without folding into a shuffle.140define <32 x i16> @mask_v32i1_lower16(<32 x i16> %a, <32 x i16> %b, <32 x i16> %c, <32 x i16> %d) {141; AVX512F-LABEL: mask_v32i1_lower16:142; AVX512F: # %bb.0:143; AVX512F-NEXT: vextracti64x4 $1, %zmm3, %ymm3144; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm2145; AVX512F-NEXT: vpcmpgtw %ymm3, %ymm2, %ymm2146; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 = -1147; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2148; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm2 & (zmm0 ^ zmm1))149; AVX512F-NEXT: retq150;151; AVX512DQ-LABEL: mask_v32i1_lower16:152; AVX512DQ: # %bb.0:153; AVX512DQ-NEXT: vextracti64x4 $1, %zmm3, %ymm3154; AVX512DQ-NEXT: vextracti64x4 $1, %zmm2, %ymm2155; AVX512DQ-NEXT: vpcmpgtw %ymm3, %ymm2, %ymm2156; AVX512DQ-NEXT: vpternlogd {{.*#+}} zmm3 = -1157; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2158; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm2 & (zmm0 ^ zmm1))159; AVX512DQ-NEXT: retq160;161; AVX512BW-LABEL: mask_v32i1_lower16:162; AVX512BW: # %bb.0:163; AVX512BW-NEXT: movl $65535, %eax # imm = 0xFFFF164; AVX512BW-NEXT: kmovd %eax, %k0165; AVX512BW-NEXT: vpcmpgtw %zmm3, %zmm2, %k1166; AVX512BW-NEXT: kord %k0, %k1, %k1167; AVX512BW-NEXT: vpblendmw %zmm0, %zmm1, %zmm0 {%k1}168; AVX512BW-NEXT: retq169;170; AVX512DQBW-LABEL: mask_v32i1_lower16:171; AVX512DQBW: # %bb.0:172; AVX512DQBW-NEXT: kxnorw %k0, %k0, %k0173; AVX512DQBW-NEXT: vpcmpgtw %zmm3, %zmm2, %k1174; AVX512DQBW-NEXT: kord %k0, %k1, %k1175; AVX512DQBW-NEXT: vpblendmw %zmm0, %zmm1, %zmm0 {%k1}176; AVX512DQBW-NEXT: retq177 %mask0 = bitcast i32 65535 to <32 x i1>178 %mask1 = icmp sgt <32 x i16> %c, %d179 %mask = or <32 x i1> %mask0, %mask1180 %res = select <32 x i1> %mask, <32 x i16> %a, <32 x i16> %b181 ret <32 x i16> %res182}183 184; Test case 6: v64i1 mask via bitconvert combined with dynamic condition.185; Verifies the KSET1D submask pattern survives past SelectionDAG combines.186define <64 x i8> @mask_v64i1_lower32(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {187; AVX512F-LABEL: mask_v64i1_lower32:188; AVX512F: # %bb.0:189; AVX512F-NEXT: vextracti64x4 $1, %zmm3, %ymm3190; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm2191; AVX512F-NEXT: vpcmpgtb %ymm3, %ymm2, %ymm2192; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 = -1193; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2194; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm2 & (zmm0 ^ zmm1))195; AVX512F-NEXT: retq196;197; AVX512DQ-LABEL: mask_v64i1_lower32:198; AVX512DQ: # %bb.0:199; AVX512DQ-NEXT: vextracti64x4 $1, %zmm3, %ymm3200; AVX512DQ-NEXT: vextracti64x4 $1, %zmm2, %ymm2201; AVX512DQ-NEXT: vpcmpgtb %ymm3, %ymm2, %ymm2202; AVX512DQ-NEXT: vpternlogd {{.*#+}} zmm3 = -1203; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2204; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm0 = zmm1 ^ (zmm2 & (zmm0 ^ zmm1))205; AVX512DQ-NEXT: retq206;207; AVX512BW-LABEL: mask_v64i1_lower32:208; AVX512BW: # %bb.0:209; AVX512BW-NEXT: movl $4294967295, %eax # imm = 0xFFFFFFFF210; AVX512BW-NEXT: kmovq %rax, %k0211; AVX512BW-NEXT: vpcmpgtb %zmm3, %zmm2, %k1212; AVX512BW-NEXT: korq %k0, %k1, %k1213; AVX512BW-NEXT: vpblendmb %zmm0, %zmm1, %zmm0 {%k1}214; AVX512BW-NEXT: retq215;216; AVX512DQBW-LABEL: mask_v64i1_lower32:217; AVX512DQBW: # %bb.0:218; AVX512DQBW-NEXT: kxnord %k0, %k0, %k0219; AVX512DQBW-NEXT: vpcmpgtb %zmm3, %zmm2, %k1220; AVX512DQBW-NEXT: korq %k0, %k1, %k1221; AVX512DQBW-NEXT: vpblendmb %zmm0, %zmm1, %zmm0 {%k1}222; AVX512DQBW-NEXT: retq223 %mask0 = bitcast i64 4294967295 to <64 x i1>224 %mask1 = icmp sgt <64 x i8> %c, %d225 %mask = or <64 x i1> %mask0, %mask1226 %res = select <64 x i1> %mask, <64 x i8> %a, <64 x i8> %b227 ret <64 x i8> %res228}229 230