645 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=X64-AVX5123; RUN: llc < %s -mtriple=x86_64-- -mcpu=knl | FileCheck %s --check-prefixes=X64-KNL4 5define <2 x i1> @i8_mask_extract2(i8 %mask) {6; X64-AVX512-LABEL: i8_mask_extract2:7; X64-AVX512: # %bb.0:8; X64-AVX512-NEXT: kmovd %edi, %k09; X64-AVX512-NEXT: vpmovm2q %k0, %xmm010; X64-AVX512-NEXT: retq11;12; X64-KNL-LABEL: i8_mask_extract2:13; X64-KNL: # %bb.0:14; X64-KNL-NEXT: vmovd %edi, %xmm015; X64-KNL-NEXT: vpbroadcastw {{.*#+}} xmm1 = [1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2]16; X64-KNL-NEXT: vpbroadcastb %xmm0, %xmm017; X64-KNL-NEXT: vpand %xmm1, %xmm0, %xmm018; X64-KNL-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm019; X64-KNL-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero20; X64-KNL-NEXT: retq21 %.splatinsert = insertelement <2 x i8> poison, i8 %mask, i64 022 %.splat = shufflevector <2 x i8> %.splatinsert, <2 x i8> poison, <2 x i32> zeroinitializer23 %1 = and <2 x i8> %.splat, <i8 1, i8 2>24 %cmp.45 = icmp ne <2 x i8> %1, zeroinitializer25 ret <2 x i1> %cmp.4526}27 28define <2 x i1> @invert_i8_mask_extract2(i8 %mask) {29; X64-AVX512-LABEL: invert_i8_mask_extract2:30; X64-AVX512: # %bb.0:31; X64-AVX512-NEXT: kmovd %edi, %k032; X64-AVX512-NEXT: knotw %k0, %k033; X64-AVX512-NEXT: vpmovm2q %k0, %xmm034; X64-AVX512-NEXT: retq35;36; X64-KNL-LABEL: invert_i8_mask_extract2:37; X64-KNL: # %bb.0:38; X64-KNL-NEXT: vmovd %edi, %xmm039; X64-KNL-NEXT: vpbroadcastb %xmm0, %xmm040; X64-KNL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm041; X64-KNL-NEXT: vpxor %xmm1, %xmm1, %xmm142; X64-KNL-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm043; X64-KNL-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero44; X64-KNL-NEXT: retq45 %.splatinsert = insertelement <2 x i8> poison, i8 %mask, i64 046 %.splat = shufflevector <2 x i8> %.splatinsert, <2 x i8> poison, <2 x i32> zeroinitializer47 %1 = and <2 x i8> %.splat, <i8 1, i8 2>48 %cmp.45 = icmp eq <2 x i8> %1, zeroinitializer49 ret <2 x i1> %cmp.4550}51 52define <4 x i1> @i8_mask_extract_4(i8 %mask) {53; X64-AVX512-LABEL: i8_mask_extract_4:54; X64-AVX512: # %bb.0:55; X64-AVX512-NEXT: kmovd %edi, %k056; X64-AVX512-NEXT: vpmovm2d %k0, %xmm057; X64-AVX512-NEXT: retq58;59; X64-KNL-LABEL: i8_mask_extract_4:60; X64-KNL: # %bb.0:61; X64-KNL-NEXT: vmovd %edi, %xmm062; X64-KNL-NEXT: vpbroadcastb %xmm0, %xmm063; X64-KNL-NEXT: vpbroadcastd {{.*#+}} xmm1 = [1,2,4,8,1,2,4,8,1,2,4,8,1,2,4,8]64; X64-KNL-NEXT: vpand %xmm1, %xmm0, %xmm065; X64-KNL-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm066; X64-KNL-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero67; X64-KNL-NEXT: retq68 %.splatinsert = insertelement <4 x i8> poison, i8 %mask, i64 069 %.splat = shufflevector <4 x i8> %.splatinsert, <4 x i8> poison, <4 x i32> zeroinitializer70 %1 = and <4 x i8> %.splat, <i8 1, i8 2, i8 4, i8 8>71 %cmp.45 = icmp ne <4 x i8> %1, zeroinitializer72 ret <4 x i1> %cmp.4573}74 75define <4 x i1> @invert_i8_mask_extract_4(i8 %mask) {76; X64-AVX512-LABEL: invert_i8_mask_extract_4:77; X64-AVX512: # %bb.0:78; X64-AVX512-NEXT: kmovd %edi, %k079; X64-AVX512-NEXT: knotw %k0, %k080; X64-AVX512-NEXT: vpmovm2d %k0, %xmm081; X64-AVX512-NEXT: retq82;83; X64-KNL-LABEL: invert_i8_mask_extract_4:84; X64-KNL: # %bb.0:85; X64-KNL-NEXT: vmovd %edi, %xmm086; X64-KNL-NEXT: vpbroadcastb %xmm0, %xmm087; X64-KNL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm088; X64-KNL-NEXT: vpxor %xmm1, %xmm1, %xmm189; X64-KNL-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm090; X64-KNL-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero91; X64-KNL-NEXT: retq92 %.splatinsert = insertelement <4 x i8> poison, i8 %mask, i64 093 %.splat = shufflevector <4 x i8> %.splatinsert, <4 x i8> poison, <4 x i32> zeroinitializer94 %1 = and <4 x i8> %.splat, <i8 1, i8 2, i8 4, i8 8>95 %cmp.45 = icmp eq <4 x i8> %1, zeroinitializer96 ret <4 x i1> %cmp.4597}98 99define <8 x i1> @i8_mask_extract_8(i8 %mask) {100; X64-AVX512-LABEL: i8_mask_extract_8:101; X64-AVX512: # %bb.0:102; X64-AVX512-NEXT: kmovd %edi, %k0103; X64-AVX512-NEXT: vpmovm2w %k0, %xmm0104; X64-AVX512-NEXT: retq105;106; X64-KNL-LABEL: i8_mask_extract_8:107; X64-KNL: # %bb.0:108; X64-KNL-NEXT: vmovd %edi, %xmm0109; X64-KNL-NEXT: vpbroadcastb %xmm0, %xmm0110; X64-KNL-NEXT: vpbroadcastq {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]111; X64-KNL-NEXT: vpand %xmm1, %xmm0, %xmm0112; X64-KNL-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0113; X64-KNL-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero114; X64-KNL-NEXT: retq115 %.splatinsert = insertelement <8 x i8> poison, i8 %mask, i64 0116 %.splat = shufflevector <8 x i8> %.splatinsert, <8 x i8> poison, <8 x i32> zeroinitializer117 %1 = and <8 x i8> %.splat, <i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128>118 %cmp.45 = icmp ne <8 x i8> %1, zeroinitializer119 ret <8 x i1> %cmp.45120}121 122define <8 x i1> @invert_i8_mask_extract_8(i8 %mask) {123; X64-AVX512-LABEL: invert_i8_mask_extract_8:124; X64-AVX512: # %bb.0:125; X64-AVX512-NEXT: kmovd %edi, %k0126; X64-AVX512-NEXT: knotb %k0, %k0127; X64-AVX512-NEXT: vpmovm2w %k0, %xmm0128; X64-AVX512-NEXT: retq129;130; X64-KNL-LABEL: invert_i8_mask_extract_8:131; X64-KNL: # %bb.0:132; X64-KNL-NEXT: vmovd %edi, %xmm0133; X64-KNL-NEXT: vpbroadcastb %xmm0, %xmm0134; X64-KNL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0135; X64-KNL-NEXT: vpxor %xmm1, %xmm1, %xmm1136; X64-KNL-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0137; X64-KNL-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero138; X64-KNL-NEXT: retq139 %.splatinsert = insertelement <8 x i8> poison, i8 %mask, i64 0140 %.splat = shufflevector <8 x i8> %.splatinsert, <8 x i8> poison, <8 x i32> zeroinitializer141 %1 = and <8 x i8> %.splat, <i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128>142 %cmp.45 = icmp eq <8 x i8> %1, zeroinitializer143 ret <8 x i1> %cmp.45144}145 146define <8 x i1> @i8_mask_extract_7(i8 %mask) {147; X64-AVX512-LABEL: i8_mask_extract_7:148; X64-AVX512: # %bb.0:149; X64-AVX512-NEXT: shrb %dil150; X64-AVX512-NEXT: movzbl %dil, %eax151; X64-AVX512-NEXT: kmovd %eax, %k0152; X64-AVX512-NEXT: vpmovm2w %k0, %xmm0153; X64-AVX512-NEXT: retq154;155; X64-KNL-LABEL: i8_mask_extract_7:156; X64-KNL: # %bb.0:157; X64-KNL-NEXT: vmovd %edi, %xmm0158; X64-KNL-NEXT: vpbroadcastb %xmm0, %xmm0159; X64-KNL-NEXT: vpbroadcastq {{.*#+}} xmm1 = [2,4,8,16,32,64,128,0,2,4,8,16,32,64,128,0]160; X64-KNL-NEXT: vpand %xmm1, %xmm0, %xmm0161; X64-KNL-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0162; X64-KNL-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero163; X64-KNL-NEXT: retq164 %.splatinsert = insertelement <8 x i8> poison, i8 %mask, i64 0165 %.splat = shufflevector <8 x i8> %.splatinsert, <8 x i8> poison, <8 x i32> zeroinitializer166 %1 = and <8 x i8> %.splat, <i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 poison>167 %cmp.45 = icmp ne <8 x i8> %1, zeroinitializer168 ret <8 x i1> %cmp.45169}170 171define <8 x i1> @invert_i8_mask_extract_7(i8 %mask) {172; X64-AVX512-LABEL: invert_i8_mask_extract_7:173; X64-AVX512: # %bb.0:174; X64-AVX512-NEXT: shrb %dil175; X64-AVX512-NEXT: movzbl %dil, %eax176; X64-AVX512-NEXT: kmovd %eax, %k0177; X64-AVX512-NEXT: knotb %k0, %k0178; X64-AVX512-NEXT: vpmovm2w %k0, %xmm0179; X64-AVX512-NEXT: retq180;181; X64-KNL-LABEL: invert_i8_mask_extract_7:182; X64-KNL: # %bb.0:183; X64-KNL-NEXT: vmovd %edi, %xmm0184; X64-KNL-NEXT: vpbroadcastb %xmm0, %xmm0185; X64-KNL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0186; X64-KNL-NEXT: vpxor %xmm1, %xmm1, %xmm1187; X64-KNL-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0188; X64-KNL-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero189; X64-KNL-NEXT: retq190 %.splatinsert = insertelement <8 x i8> poison, i8 %mask, i64 0191 %.splat = shufflevector <8 x i8> %.splatinsert, <8 x i8> poison, <8 x i32> zeroinitializer192 %1 = and <8 x i8> %.splat, <i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 128, i8 poison>193 %cmp.45 = icmp eq <8 x i8> %1, zeroinitializer194 ret <8 x i1> %cmp.45195}196 197define <4 x i1> @i16_mask_extract_4(i16 %mask) {198; X64-AVX512-LABEL: i16_mask_extract_4:199; X64-AVX512: # %bb.0:200; X64-AVX512-NEXT: kmovd %edi, %k0201; X64-AVX512-NEXT: vpmovm2d %k0, %xmm0202; X64-AVX512-NEXT: retq203;204; X64-KNL-LABEL: i16_mask_extract_4:205; X64-KNL: # %bb.0:206; X64-KNL-NEXT: vmovd %edi, %xmm0207; X64-KNL-NEXT: vpbroadcastw %xmm0, %xmm0208; X64-KNL-NEXT: vpbroadcastq {{.*#+}} xmm1 = [1,2,4,8,1,2,4,8]209; X64-KNL-NEXT: vpand %xmm1, %xmm0, %xmm0210; X64-KNL-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0211; X64-KNL-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero212; X64-KNL-NEXT: retq213 %.splatinsert = insertelement <4 x i16> poison, i16 %mask, i64 0214 %.splat = shufflevector <4 x i16> %.splatinsert, <4 x i16> poison, <4 x i32> zeroinitializer215 %1 = and <4 x i16> %.splat, <i16 1, i16 2, i16 4, i16 8>216 %cmp.45 = icmp ne <4 x i16> %1, zeroinitializer217 ret <4 x i1> %cmp.45218}219 220define <4 x i1> @invert_i16_mask_extract_4(i16 %mask) {221; X64-AVX512-LABEL: invert_i16_mask_extract_4:222; X64-AVX512: # %bb.0:223; X64-AVX512-NEXT: kmovd %edi, %k0224; X64-AVX512-NEXT: knotw %k0, %k0225; X64-AVX512-NEXT: vpmovm2d %k0, %xmm0226; X64-AVX512-NEXT: retq227;228; X64-KNL-LABEL: invert_i16_mask_extract_4:229; X64-KNL: # %bb.0:230; X64-KNL-NEXT: vmovd %edi, %xmm0231; X64-KNL-NEXT: vpbroadcastw %xmm0, %xmm0232; X64-KNL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0233; X64-KNL-NEXT: vpxor %xmm1, %xmm1, %xmm1234; X64-KNL-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0235; X64-KNL-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero236; X64-KNL-NEXT: retq237 %.splatinsert = insertelement <4 x i16> poison, i16 %mask, i64 0238 %.splat = shufflevector <4 x i16> %.splatinsert, <4 x i16> poison, <4 x i32> zeroinitializer239 %1 = and <4 x i16> %.splat, <i16 1, i16 2, i16 4, i16 8>240 %cmp.45 = icmp eq <4 x i16> %1, zeroinitializer241 ret <4 x i1> %cmp.45242}243 244define <8 x i1> @i16_mask_extract_8(i16 %mask) {245; X64-AVX512-LABEL: i16_mask_extract_8:246; X64-AVX512: # %bb.0:247; X64-AVX512-NEXT: vpbroadcastw %edi, %xmm0248; X64-AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]249; X64-AVX512-NEXT: vpand %xmm1, %xmm0, %xmm0250; X64-AVX512-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0251; X64-AVX512-NEXT: retq252;253; X64-KNL-LABEL: i16_mask_extract_8:254; X64-KNL: # %bb.0:255; X64-KNL-NEXT: vmovd %edi, %xmm0256; X64-KNL-NEXT: vpbroadcastw %xmm0, %xmm0257; X64-KNL-NEXT: vmovdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]258; X64-KNL-NEXT: vpand %xmm1, %xmm0, %xmm0259; X64-KNL-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0260; X64-KNL-NEXT: retq261 %.splatinsert = insertelement <8 x i16> poison, i16 %mask, i64 0262 %.splat = shufflevector <8 x i16> %.splatinsert, <8 x i16> poison, <8 x i32> zeroinitializer263 %1 = and <8 x i16> %.splat, <i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128>264 %cmp.45 = icmp ne <8 x i16> %1, zeroinitializer265 ret <8 x i1> %cmp.45266}267 268define <8 x i1> @invert_i16_mask_extract_8(i16 %mask) {269; X64-AVX512-LABEL: invert_i16_mask_extract_8:270; X64-AVX512: # %bb.0:271; X64-AVX512-NEXT: vpbroadcastw %edi, %xmm0272; X64-AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0273; X64-AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1274; X64-AVX512-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0275; X64-AVX512-NEXT: retq276;277; X64-KNL-LABEL: invert_i16_mask_extract_8:278; X64-KNL: # %bb.0:279; X64-KNL-NEXT: vmovd %edi, %xmm0280; X64-KNL-NEXT: vpbroadcastw %xmm0, %xmm0281; X64-KNL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0282; X64-KNL-NEXT: vpxor %xmm1, %xmm1, %xmm1283; X64-KNL-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0284; X64-KNL-NEXT: retq285 %.splatinsert = insertelement <8 x i16> poison, i16 %mask, i64 0286 %.splat = shufflevector <8 x i16> %.splatinsert, <8 x i16> poison, <8 x i32> zeroinitializer287 %1 = and <8 x i16> %.splat, <i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128>288 %cmp.45 = icmp eq <8 x i16> %1, zeroinitializer289 ret <8 x i1> %cmp.45290}291 292define <16 x i1> @i16_mask_extract_16(i16 %mask) {293; X64-AVX512-LABEL: i16_mask_extract_16:294; X64-AVX512: # %bb.0:295; X64-AVX512-NEXT: kmovd %edi, %k0296; X64-AVX512-NEXT: vpmovm2b %k0, %xmm0297; X64-AVX512-NEXT: retq298;299; X64-KNL-LABEL: i16_mask_extract_16:300; X64-KNL: # %bb.0:301; X64-KNL-NEXT: vmovd %edi, %xmm0302; X64-KNL-NEXT: vpbroadcastw %xmm0, %ymm0303; X64-KNL-NEXT: vmovdqa {{.*#+}} ymm1 = [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]304; X64-KNL-NEXT: vpand %ymm1, %ymm0, %ymm0305; X64-KNL-NEXT: vpcmpeqw %ymm1, %ymm0, %ymm0306; X64-KNL-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero307; X64-KNL-NEXT: vpmovdb %zmm0, %xmm0308; X64-KNL-NEXT: retq309 %.splatinsert = insertelement <16 x i16> poison, i16 %mask, i64 0310 %.splat = shufflevector <16 x i16> %.splatinsert, <16 x i16> poison, <16 x i32> zeroinitializer311 %1 = and <16 x i16> %.splat, <i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 512, i16 1024, i16 2048, i16 4096, i16 8192, i16 16384, i16 32768>312 %cmp.45 = icmp ne <16 x i16> %1, zeroinitializer313 ret <16 x i1> %cmp.45314}315 316define <16 x i1> @invert_i16_mask_extract_16(i16 %mask) {317; X64-AVX512-LABEL: invert_i16_mask_extract_16:318; X64-AVX512: # %bb.0:319; X64-AVX512-NEXT: kmovd %edi, %k0320; X64-AVX512-NEXT: knotw %k0, %k0321; X64-AVX512-NEXT: vpmovm2b %k0, %xmm0322; X64-AVX512-NEXT: retq323;324; X64-KNL-LABEL: invert_i16_mask_extract_16:325; X64-KNL: # %bb.0:326; X64-KNL-NEXT: vmovd %edi, %xmm0327; X64-KNL-NEXT: vpbroadcastw %xmm0, %ymm0328; X64-KNL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0329; X64-KNL-NEXT: vpxor %xmm1, %xmm1, %xmm1330; X64-KNL-NEXT: vpcmpeqw %ymm1, %ymm0, %ymm0331; X64-KNL-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero332; X64-KNL-NEXT: vpmovdb %zmm0, %xmm0333; X64-KNL-NEXT: retq334 %.splatinsert = insertelement <16 x i16> poison, i16 %mask, i64 0335 %.splat = shufflevector <16 x i16> %.splatinsert, <16 x i16> poison, <16 x i32> zeroinitializer336 %1 = and <16 x i16> %.splat, <i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 512, i16 1024, i16 2048, i16 4096, i16 8192, i16 16384, i16 32768>337 %cmp.45 = icmp eq <16 x i16> %1, zeroinitializer338 ret <16 x i1> %cmp.45339}340 341define <16 x i1> @i32_mask_extract_16(i32 %mask) {342; X64-AVX512-LABEL: i32_mask_extract_16:343; X64-AVX512: # %bb.0:344; X64-AVX512-NEXT: kmovd %edi, %k0345; X64-AVX512-NEXT: vpmovm2b %k0, %xmm0346; X64-AVX512-NEXT: retq347;348; X64-KNL-LABEL: i32_mask_extract_16:349; X64-KNL: # %bb.0:350; X64-KNL-NEXT: kmovw %edi, %k1351; X64-KNL-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1352; X64-KNL-NEXT: vpmovdb %zmm0, %xmm0353; X64-KNL-NEXT: retq354 %.splatinsert = insertelement <16 x i32> poison, i32 %mask, i64 0355 %.splat = shufflevector <16 x i32> %.splatinsert, <16 x i32> poison, <16 x i32> zeroinitializer356 %1 = and <16 x i32> %.splat, <i32 1, i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256, i32 512, i32 1024, i32 2048, i32 4096, i32 8192, i32 16384, i32 32768>357 %cmp.45 = icmp ne <16 x i32> %1, zeroinitializer358 ret <16 x i1> %cmp.45359}360 361define <16 x i1> @invert_i32_mask_extract_16(i32 %mask) {362; X64-AVX512-LABEL: invert_i32_mask_extract_16:363; X64-AVX512: # %bb.0:364; X64-AVX512-NEXT: kmovd %edi, %k0365; X64-AVX512-NEXT: knotw %k0, %k0366; X64-AVX512-NEXT: vpmovm2b %k0, %xmm0367; X64-AVX512-NEXT: retq368;369; X64-KNL-LABEL: invert_i32_mask_extract_16:370; X64-KNL: # %bb.0:371; X64-KNL-NEXT: kmovw %edi, %k0372; X64-KNL-NEXT: knotw %k0, %k1373; X64-KNL-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1374; X64-KNL-NEXT: vpmovdb %zmm0, %xmm0375; X64-KNL-NEXT: retq376 %.splatinsert = insertelement <16 x i32> poison, i32 %mask, i64 0377 %.splat = shufflevector <16 x i32> %.splatinsert, <16 x i32> poison, <16 x i32> zeroinitializer378 %1 = and <16 x i32> %.splat, <i32 1, i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256, i32 512, i32 1024, i32 2048, i32 4096, i32 8192, i32 16384, i32 32768>379 %cmp.45 = icmp eq <16 x i32> %1, zeroinitializer380 ret <16 x i1> %cmp.45381}382 383define <32 x i1> @i32_mask_extract_32(i32 %mask) {384; X64-AVX512-LABEL: i32_mask_extract_32:385; X64-AVX512: # %bb.0:386; X64-AVX512-NEXT: kmovd %edi, %k0387; X64-AVX512-NEXT: kshiftrd $16, %k0, %k1388; X64-AVX512-NEXT: kunpckwd %k0, %k1, %k0389; X64-AVX512-NEXT: vpmovm2b %k0, %ymm0390; X64-AVX512-NEXT: retq391;392; X64-KNL-LABEL: i32_mask_extract_32:393; X64-KNL: # %bb.0:394; X64-KNL-NEXT: kmovw %edi, %k1395; X64-KNL-NEXT: shrl $16, %edi396; X64-KNL-NEXT: kmovw %edi, %k2397; X64-KNL-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1398; X64-KNL-NEXT: vpmovdb %zmm0, %xmm0399; X64-KNL-NEXT: vpternlogd {{.*#+}} zmm1 {%k2} {z} = -1400; X64-KNL-NEXT: vpmovdb %zmm1, %xmm1401; X64-KNL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0402; X64-KNL-NEXT: retq403 %.splatinsert = insertelement <32 x i32> poison, i32 %mask, i64 0404 %.splat = shufflevector <32 x i32> %.splatinsert, <32 x i32> poison, <32 x i32> zeroinitializer405 %1 = and <32 x i32> %.splat, <i32 1, i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256, i32 512, i32 1024, i32 2048, i32 4096, i32 8192, i32 16384, i32 32768, i32 65536, i32 131072, i32 262144, i32 524288, i32 1048576, i32 2097152, i32 4194304, i32 8388608, i32 16777216, i32 33554432, i32 67108864, i32 134217728, i32 268435456, i32 536870912, i32 1073741824, i32 2147483648>406 %cmp.45 = icmp ne <32 x i32> %1, zeroinitializer407 ret <32 x i1> %cmp.45408}409 410define <32 x i1> @invert_i32_mask_extract_32(i32 %mask) {411; X64-AVX512-LABEL: invert_i32_mask_extract_32:412; X64-AVX512: # %bb.0:413; X64-AVX512-NEXT: kmovd %edi, %k0414; X64-AVX512-NEXT: kshiftrd $16, %k0, %k1415; X64-AVX512-NEXT: kunpckwd %k0, %k1, %k0416; X64-AVX512-NEXT: vpmovm2b %k0, %ymm0417; X64-AVX512-NEXT: retq418;419; X64-KNL-LABEL: invert_i32_mask_extract_32:420; X64-KNL: # %bb.0:421; X64-KNL-NEXT: kmovw %edi, %k1422; X64-KNL-NEXT: shrl $16, %edi423; X64-KNL-NEXT: kmovw %edi, %k2424; X64-KNL-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1425; X64-KNL-NEXT: vpmovdb %zmm0, %xmm0426; X64-KNL-NEXT: vpternlogd {{.*#+}} zmm1 {%k2} {z} = -1427; X64-KNL-NEXT: vpmovdb %zmm1, %xmm1428; X64-KNL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0429; X64-KNL-NEXT: retq430 %.splatinsert = insertelement <32 x i32> poison, i32 %mask, i64 0431 %.splat = shufflevector <32 x i32> %.splatinsert, <32 x i32> poison, <32 x i32> zeroinitializer432 %1 = and <32 x i32> %.splat, <i32 1, i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256, i32 512, i32 1024, i32 2048, i32 4096, i32 8192, i32 16384, i32 32768, i32 65536, i32 131072, i32 262144, i32 524288, i32 1048576, i32 2097152, i32 4194304, i32 8388608, i32 16777216, i32 33554432, i32 67108864, i32 134217728, i32 268435456, i32 536870912, i32 1073741824, i32 2147483648>433 %cmp.45 = icmp ne <32 x i32> %1, zeroinitializer434 ret <32 x i1> %cmp.45435}436 437define <32 x i1> @i64_mask_extract_32(i64 %mask) {438; X64-AVX512-LABEL: i64_mask_extract_32:439; X64-AVX512: # %bb.0:440; X64-AVX512-NEXT: kmovq %rdi, %k0441; X64-AVX512-NEXT: kshiftrd $8, %k0, %k1442; X64-AVX512-NEXT: kunpckbw %k0, %k1, %k1443; X64-AVX512-NEXT: kshiftrd $16, %k0, %k2444; X64-AVX512-NEXT: kshiftrd $24, %k0, %k0445; X64-AVX512-NEXT: kunpckbw %k2, %k0, %k0446; X64-AVX512-NEXT: kunpckwd %k1, %k0, %k0447; X64-AVX512-NEXT: vpmovm2b %k0, %ymm0448; X64-AVX512-NEXT: retq449;450; X64-KNL-LABEL: i64_mask_extract_32:451; X64-KNL: # %bb.0:452; X64-KNL-NEXT: movl %edi, %eax453; X64-KNL-NEXT: shrl $16, %eax454; X64-KNL-NEXT: kmovw %eax, %k0455; X64-KNL-NEXT: movl %edi, %eax456; X64-KNL-NEXT: shrl $24, %eax457; X64-KNL-NEXT: kmovw %eax, %k1458; X64-KNL-NEXT: kunpckbw %k0, %k1, %k1459; X64-KNL-NEXT: kmovw %edi, %k0460; X64-KNL-NEXT: shrl $8, %edi461; X64-KNL-NEXT: kmovw %edi, %k2462; X64-KNL-NEXT: kunpckbw %k0, %k2, %k2463; X64-KNL-NEXT: vpternlogd {{.*#+}} zmm0 {%k2} {z} = -1464; X64-KNL-NEXT: vpmovdb %zmm0, %xmm0465; X64-KNL-NEXT: vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1466; X64-KNL-NEXT: vpmovdb %zmm1, %xmm1467; X64-KNL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0468; X64-KNL-NEXT: retq469 %.splatinsert = insertelement <32 x i64> poison, i64 %mask, i64 0470 %.splat = shufflevector <32 x i64> %.splatinsert, <32 x i64> poison, <32 x i32> zeroinitializer471 %1 = and <32 x i64> %.splat, <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 256, i64 512, i64 1024, i64 2048, i64 4096, i64 8192, i64 16384, i64 32768, i64 65536, i64 131072, i64 262144, i64 524288, i64 1048576, i64 2097152, i64 4194304, i64 8388608, i64 16777216, i64 33554432, i64 67108864, i64 134217728, i64 268435456, i64 536870912, i64 1073741824, i64 2147483648>472 %cmp.45 = icmp ne <32 x i64> %1, zeroinitializer473 ret <32 x i1> %cmp.45474}475 476define <32 x i1> @invert_i64_mask_extract_32(i64 %mask) {477; X64-AVX512-LABEL: invert_i64_mask_extract_32:478; X64-AVX512: # %bb.0:479; X64-AVX512-NEXT: kmovq %rdi, %k0480; X64-AVX512-NEXT: kshiftrd $8, %k0, %k1481; X64-AVX512-NEXT: kunpckbw %k0, %k1, %k1482; X64-AVX512-NEXT: kshiftrd $16, %k0, %k2483; X64-AVX512-NEXT: kshiftrd $24, %k0, %k0484; X64-AVX512-NEXT: kunpckbw %k2, %k0, %k0485; X64-AVX512-NEXT: kunpckwd %k1, %k0, %k0486; X64-AVX512-NEXT: knotd %k0, %k0487; X64-AVX512-NEXT: vpmovm2b %k0, %ymm0488; X64-AVX512-NEXT: retq489;490; X64-KNL-LABEL: invert_i64_mask_extract_32:491; X64-KNL: # %bb.0:492; X64-KNL-NEXT: movl %edi, %eax493; X64-KNL-NEXT: shrl $16, %eax494; X64-KNL-NEXT: kmovw %eax, %k0495; X64-KNL-NEXT: movl %edi, %eax496; X64-KNL-NEXT: shrl $24, %eax497; X64-KNL-NEXT: kmovw %eax, %k1498; X64-KNL-NEXT: kunpckbw %k0, %k1, %k0499; X64-KNL-NEXT: knotw %k0, %k1500; X64-KNL-NEXT: kmovw %edi, %k0501; X64-KNL-NEXT: shrl $8, %edi502; X64-KNL-NEXT: kmovw %edi, %k2503; X64-KNL-NEXT: kunpckbw %k0, %k2, %k0504; X64-KNL-NEXT: knotw %k0, %k2505; X64-KNL-NEXT: vpternlogd {{.*#+}} zmm0 {%k2} {z} = -1506; X64-KNL-NEXT: vpmovdb %zmm0, %xmm0507; X64-KNL-NEXT: vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1508; X64-KNL-NEXT: vpmovdb %zmm1, %xmm1509; X64-KNL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0510; X64-KNL-NEXT: retq511 %.splatinsert = insertelement <32 x i64> poison, i64 %mask, i64 0512 %.splat = shufflevector <32 x i64> %.splatinsert, <32 x i64> poison, <32 x i32> zeroinitializer513 %1 = and <32 x i64> %.splat, <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 256, i64 512, i64 1024, i64 2048, i64 4096, i64 8192, i64 16384, i64 32768, i64 65536, i64 131072, i64 262144, i64 524288, i64 1048576, i64 2097152, i64 4194304, i64 8388608, i64 16777216, i64 33554432, i64 67108864, i64 134217728, i64 268435456, i64 536870912, i64 1073741824, i64 2147483648>514 %cmp.45 = icmp eq <32 x i64> %1, zeroinitializer515 ret <32 x i1> %cmp.45516}517 518define <64 x i1> @i64_mask_extract_64(i64 %mask) {519; X64-AVX512-LABEL: i64_mask_extract_64:520; X64-AVX512: # %bb.0:521; X64-AVX512-NEXT: kmovq %rdi, %k0522; X64-AVX512-NEXT: kshiftrq $32, %k0, %k1523; X64-AVX512-NEXT: kshiftrq $40, %k0, %k2524; X64-AVX512-NEXT: kunpckbw %k1, %k2, %k1525; X64-AVX512-NEXT: kshiftrq $48, %k0, %k2526; X64-AVX512-NEXT: kshiftrq $56, %k0, %k3527; X64-AVX512-NEXT: kunpckbw %k2, %k3, %k2528; X64-AVX512-NEXT: kunpckwd %k1, %k2, %k1529; X64-AVX512-NEXT: kshiftrd $8, %k0, %k2530; X64-AVX512-NEXT: kunpckbw %k0, %k2, %k2531; X64-AVX512-NEXT: kshiftrd $16, %k0, %k3532; X64-AVX512-NEXT: kshiftrd $24, %k0, %k0533; X64-AVX512-NEXT: kunpckbw %k3, %k0, %k0534; X64-AVX512-NEXT: kunpckwd %k2, %k0, %k0535; X64-AVX512-NEXT: kunpckdq %k0, %k1, %k0536; X64-AVX512-NEXT: vpmovm2b %k0, %zmm0537; X64-AVX512-NEXT: retq538;539; X64-KNL-LABEL: i64_mask_extract_64:540; X64-KNL: # %bb.0:541; X64-KNL-NEXT: movq %rdi, %rax542; X64-KNL-NEXT: kmovw %esi, %k0543; X64-KNL-NEXT: movl %esi, %ecx544; X64-KNL-NEXT: shrl $8, %ecx545; X64-KNL-NEXT: kmovw %ecx, %k1546; X64-KNL-NEXT: kunpckbw %k0, %k1, %k0547; X64-KNL-NEXT: movl %esi, %ecx548; X64-KNL-NEXT: shrl $16, %ecx549; X64-KNL-NEXT: kmovw %ecx, %k1550; X64-KNL-NEXT: movl %esi, %ecx551; X64-KNL-NEXT: shrl $24, %ecx552; X64-KNL-NEXT: kmovw %ecx, %k2553; X64-KNL-NEXT: kunpckbw %k1, %k2, %k1554; X64-KNL-NEXT: movq %rsi, %rcx555; X64-KNL-NEXT: shrq $32, %rcx556; X64-KNL-NEXT: kmovw %ecx, %k2557; X64-KNL-NEXT: movq %rsi, %rcx558; X64-KNL-NEXT: shrq $40, %rcx559; X64-KNL-NEXT: kmovw %ecx, %k3560; X64-KNL-NEXT: kunpckbw %k2, %k3, %k2561; X64-KNL-NEXT: movq %rsi, %rcx562; X64-KNL-NEXT: shrq $48, %rcx563; X64-KNL-NEXT: kmovw %ecx, %k3564; X64-KNL-NEXT: shrq $56, %rsi565; X64-KNL-NEXT: kmovw %esi, %k4566; X64-KNL-NEXT: kunpckbw %k3, %k4, %k3567; X64-KNL-NEXT: kmovw %k3, 6(%rdi)568; X64-KNL-NEXT: kmovw %k2, 4(%rdi)569; X64-KNL-NEXT: kmovw %k1, 2(%rdi)570; X64-KNL-NEXT: kmovw %k0, (%rdi)571; X64-KNL-NEXT: retq572 %.splatinsert = insertelement <64 x i64> poison, i64 %mask, i64 0573 %.splat = shufflevector <64 x i64> %.splatinsert, <64 x i64> poison, <64 x i32> zeroinitializer574 %1 = and <64 x i64> %.splat, <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 256, i64 512, i64 1024, i64 2048, i64 4096, i64 8192, i64 16384, i64 32768, i64 65536, i64 131072, i64 262144, i64 524288, i64 1048576, i64 2097152, i64 4194304, i64 8388608, i64 16777216, i64 33554432, i64 67108864, i64 134217728, i64 268435456, i64 536870912, i64 1073741824, i64 2147483648, i64 4294967296, i64 8589934592, i64 17179869184, i64 34359738368, i64 68719476736, i64 137438953472, i64 274877906944, i64 549755813888, i64 1099511627776, i64 2199023255552, i64 4398046511104, i64 8796093022208, i64 17592186044416, i64 35184372088832, i64 70368744177664, i64 140737488355328, i64 281474976710656, i64 562949953421312, i64 1125899906842624, i64 2251799813685248, i64 4503599627370496, i64 9007199254740992, i64 18014398509481984, i64 36028797018963968, i64 72057594037927936, i64 144115188075855872, i64 288230376151711744, i64 576460752303423488, i64 1152921504606846976, i64 2305843009213693952, i64 4611686018427387904, i64 9223372036854775808>575 %cmp.45 = icmp ne <64 x i64> %1, zeroinitializer576 ret <64 x i1> %cmp.45577}578 579define <64 x i1> @invert_i64_mask_extract_64(i64 %mask) {580; X64-AVX512-LABEL: invert_i64_mask_extract_64:581; X64-AVX512: # %bb.0:582; X64-AVX512-NEXT: kmovq %rdi, %k0583; X64-AVX512-NEXT: kshiftrq $32, %k0, %k1584; X64-AVX512-NEXT: kshiftrq $40, %k0, %k2585; X64-AVX512-NEXT: kunpckbw %k1, %k2, %k1586; X64-AVX512-NEXT: kshiftrq $48, %k0, %k2587; X64-AVX512-NEXT: kshiftrq $56, %k0, %k3588; X64-AVX512-NEXT: kunpckbw %k2, %k3, %k2589; X64-AVX512-NEXT: kunpckwd %k1, %k2, %k1590; X64-AVX512-NEXT: kshiftrd $8, %k0, %k2591; X64-AVX512-NEXT: kunpckbw %k0, %k2, %k2592; X64-AVX512-NEXT: kshiftrd $16, %k0, %k3593; X64-AVX512-NEXT: kshiftrd $24, %k0, %k0594; X64-AVX512-NEXT: kunpckbw %k3, %k0, %k0595; X64-AVX512-NEXT: kunpckwd %k2, %k0, %k0596; X64-AVX512-NEXT: kunpckdq %k0, %k1, %k0597; X64-AVX512-NEXT: knotq %k0, %k0598; X64-AVX512-NEXT: vpmovm2b %k0, %zmm0599; X64-AVX512-NEXT: retq600;601; X64-KNL-LABEL: invert_i64_mask_extract_64:602; X64-KNL: # %bb.0:603; X64-KNL-NEXT: movq %rdi, %rax604; X64-KNL-NEXT: kmovw %esi, %k0605; X64-KNL-NEXT: movl %esi, %ecx606; X64-KNL-NEXT: shrl $8, %ecx607; X64-KNL-NEXT: kmovw %ecx, %k1608; X64-KNL-NEXT: kunpckbw %k0, %k1, %k0609; X64-KNL-NEXT: knotw %k0, %k0610; X64-KNL-NEXT: movl %esi, %ecx611; X64-KNL-NEXT: shrl $16, %ecx612; X64-KNL-NEXT: kmovw %ecx, %k1613; X64-KNL-NEXT: movl %esi, %ecx614; X64-KNL-NEXT: shrl $24, %ecx615; X64-KNL-NEXT: kmovw %ecx, %k2616; X64-KNL-NEXT: kunpckbw %k1, %k2, %k1617; X64-KNL-NEXT: knotw %k1, %k1618; X64-KNL-NEXT: movq %rsi, %rcx619; X64-KNL-NEXT: shrq $32, %rcx620; X64-KNL-NEXT: kmovw %ecx, %k2621; X64-KNL-NEXT: movq %rsi, %rcx622; X64-KNL-NEXT: shrq $40, %rcx623; X64-KNL-NEXT: kmovw %ecx, %k3624; X64-KNL-NEXT: kunpckbw %k2, %k3, %k2625; X64-KNL-NEXT: knotw %k2, %k2626; X64-KNL-NEXT: movq %rsi, %rcx627; X64-KNL-NEXT: shrq $48, %rcx628; X64-KNL-NEXT: kmovw %ecx, %k3629; X64-KNL-NEXT: shrq $56, %rsi630; X64-KNL-NEXT: kmovw %esi, %k4631; X64-KNL-NEXT: kunpckbw %k3, %k4, %k3632; X64-KNL-NEXT: knotw %k3, %k3633; X64-KNL-NEXT: kmovw %k3, 6(%rdi)634; X64-KNL-NEXT: kmovw %k2, 4(%rdi)635; X64-KNL-NEXT: kmovw %k1, 2(%rdi)636; X64-KNL-NEXT: kmovw %k0, (%rdi)637; X64-KNL-NEXT: retq638 %.splatinsert = insertelement <64 x i64> poison, i64 %mask, i64 0639 %.splat = shufflevector <64 x i64> %.splatinsert, <64 x i64> poison, <64 x i32> zeroinitializer640 %1 = and <64 x i64> %.splat, <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 256, i64 512, i64 1024, i64 2048, i64 4096, i64 8192, i64 16384, i64 32768, i64 65536, i64 131072, i64 262144, i64 524288, i64 1048576, i64 2097152, i64 4194304, i64 8388608, i64 16777216, i64 33554432, i64 67108864, i64 134217728, i64 268435456, i64 536870912, i64 1073741824, i64 2147483648, i64 4294967296, i64 8589934592, i64 17179869184, i64 34359738368, i64 68719476736, i64 137438953472, i64 274877906944, i64 549755813888, i64 1099511627776, i64 2199023255552, i64 4398046511104, i64 8796093022208, i64 17592186044416, i64 35184372088832, i64 70368744177664, i64 140737488355328, i64 281474976710656, i64 562949953421312, i64 1125899906842624, i64 2251799813685248, i64 4503599627370496, i64 9007199254740992, i64 18014398509481984, i64 36028797018963968, i64 72057594037927936, i64 144115188075855872, i64 288230376151711744, i64 576460752303423488, i64 1152921504606846976, i64 2305843009213693952, i64 4611686018427387904, i64 9223372036854775808>641 %cmp.45 = icmp eq <64 x i64> %1, zeroinitializer642 ret <64 x i1> %cmp.45643}644 645