4777 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_mem_shuffle2; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle %s -o - | FileCheck --check-prefixes=CHECK,CHECK-FAST %s3; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512vl,+avx512bw,+fast-variable-perlane-shuffle %s -o - | FileCheck --check-prefixes=CHECK,CHECK-FAST-PERLANE %s4 5; FIXME: All cases here should be fixed by PR343806 7define <8 x i16> @test_16xi16_to_8xi16_perm_mask0(<16 x i16> %vec) {8; CHECK-LABEL: test_16xi16_to_8xi16_perm_mask0:9; CHECK: # %bb.0:10; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm1 = [8,6,12,4,7,9,14,8]11; CHECK-NEXT: vpermw %ymm0, %ymm1, %ymm012; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm013; CHECK-NEXT: vzeroupper14; CHECK-NEXT: retq15 %res = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 8, i32 6, i32 12, i32 4, i32 7, i32 9, i32 14, i32 8>16 ret <8 x i16> %res17}18define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mask0(<16 x i16> %vec, <8 x i16> %vec2, <8 x i16> %mask) {19; CHECK-LABEL: test_masked_16xi16_to_8xi16_perm_mask0:20; CHECK: # %bb.0:21; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm122; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm3 = [8,6,12,4,7,9,14,8]23; CHECK-NEXT: vptestnmw %xmm2, %xmm2, %k124; CHECK-NEXT: vpermw %ymm0, %ymm3, %ymm1 {%k1}25; CHECK-NEXT: vmovdqa %xmm1, %xmm026; CHECK-NEXT: vzeroupper27; CHECK-NEXT: retq28 %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 8, i32 6, i32 12, i32 4, i32 7, i32 9, i32 14, i32 8>29 %cmp = icmp eq <8 x i16> %mask, zeroinitializer30 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec231 ret <8 x i16> %res32}33 34define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mask0(<16 x i16> %vec, <8 x i16> %mask) {35; CHECK-LABEL: test_masked_z_16xi16_to_8xi16_perm_mask0:36; CHECK: # %bb.0:37; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [8,6,12,4,7,9,14,8]38; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k139; CHECK-NEXT: vpermw %ymm0, %ymm2, %ymm0 {%k1} {z}40; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm041; CHECK-NEXT: vzeroupper42; CHECK-NEXT: retq43 %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 8, i32 6, i32 12, i32 4, i32 7, i32 9, i32 14, i32 8>44 %cmp = icmp eq <8 x i16> %mask, zeroinitializer45 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer46 ret <8 x i16> %res47}48define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mask1(<16 x i16> %vec, <8 x i16> %vec2, <8 x i16> %mask) {49; CHECK-LABEL: test_masked_16xi16_to_8xi16_perm_mask1:50; CHECK: # %bb.0:51; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm152; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm3 = [4,12,9,4,14,15,12,14]53; CHECK-NEXT: vptestnmw %xmm2, %xmm2, %k154; CHECK-NEXT: vpermw %ymm0, %ymm3, %ymm1 {%k1}55; CHECK-NEXT: vmovdqa %xmm1, %xmm056; CHECK-NEXT: vzeroupper57; CHECK-NEXT: retq58 %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 4, i32 12, i32 9, i32 4, i32 14, i32 15, i32 12, i32 14>59 %cmp = icmp eq <8 x i16> %mask, zeroinitializer60 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec261 ret <8 x i16> %res62}63 64define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mask1(<16 x i16> %vec, <8 x i16> %mask) {65; CHECK-LABEL: test_masked_z_16xi16_to_8xi16_perm_mask1:66; CHECK: # %bb.0:67; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [4,12,9,4,14,15,12,14]68; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k169; CHECK-NEXT: vpermw %ymm0, %ymm2, %ymm0 {%k1} {z}70; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm071; CHECK-NEXT: vzeroupper72; CHECK-NEXT: retq73 %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 4, i32 12, i32 9, i32 4, i32 14, i32 15, i32 12, i32 14>74 %cmp = icmp eq <8 x i16> %mask, zeroinitializer75 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer76 ret <8 x i16> %res77}78define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mask2(<16 x i16> %vec, <8 x i16> %vec2, <8 x i16> %mask) {79; CHECK-LABEL: test_masked_16xi16_to_8xi16_perm_mask2:80; CHECK: # %bb.0:81; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm182; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm3 = [4,11,14,10,7,1,6,9]83; CHECK-NEXT: vptestnmw %xmm2, %xmm2, %k184; CHECK-NEXT: vpermw %ymm0, %ymm3, %ymm1 {%k1}85; CHECK-NEXT: vmovdqa %xmm1, %xmm086; CHECK-NEXT: vzeroupper87; CHECK-NEXT: retq88 %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 4, i32 11, i32 14, i32 10, i32 7, i32 1, i32 6, i32 9>89 %cmp = icmp eq <8 x i16> %mask, zeroinitializer90 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec291 ret <8 x i16> %res92}93 94define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mask2(<16 x i16> %vec, <8 x i16> %mask) {95; CHECK-LABEL: test_masked_z_16xi16_to_8xi16_perm_mask2:96; CHECK: # %bb.0:97; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [4,11,14,10,7,1,6,9]98; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k199; CHECK-NEXT: vpermw %ymm0, %ymm2, %ymm0 {%k1} {z}100; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0101; CHECK-NEXT: vzeroupper102; CHECK-NEXT: retq103 %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 4, i32 11, i32 14, i32 10, i32 7, i32 1, i32 6, i32 9>104 %cmp = icmp eq <8 x i16> %mask, zeroinitializer105 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer106 ret <8 x i16> %res107}108define <8 x i16> @test_16xi16_to_8xi16_perm_mask3(<16 x i16> %vec) {109; CHECK-LABEL: test_16xi16_to_8xi16_perm_mask3:110; CHECK: # %bb.0:111; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm1 = [14,15,7,13,4,12,8,0]112; CHECK-NEXT: vpermw %ymm0, %ymm1, %ymm0113; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0114; CHECK-NEXT: vzeroupper115; CHECK-NEXT: retq116 %res = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 14, i32 15, i32 7, i32 13, i32 4, i32 12, i32 8, i32 0>117 ret <8 x i16> %res118}119define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mask3(<16 x i16> %vec, <8 x i16> %vec2, <8 x i16> %mask) {120; CHECK-LABEL: test_masked_16xi16_to_8xi16_perm_mask3:121; CHECK: # %bb.0:122; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm1123; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm3 = [14,15,7,13,4,12,8,0]124; CHECK-NEXT: vptestnmw %xmm2, %xmm2, %k1125; CHECK-NEXT: vpermw %ymm0, %ymm3, %ymm1 {%k1}126; CHECK-NEXT: vmovdqa %xmm1, %xmm0127; CHECK-NEXT: vzeroupper128; CHECK-NEXT: retq129 %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 14, i32 15, i32 7, i32 13, i32 4, i32 12, i32 8, i32 0>130 %cmp = icmp eq <8 x i16> %mask, zeroinitializer131 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2132 ret <8 x i16> %res133}134 135define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mask3(<16 x i16> %vec, <8 x i16> %mask) {136; CHECK-LABEL: test_masked_z_16xi16_to_8xi16_perm_mask3:137; CHECK: # %bb.0:138; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [14,15,7,13,4,12,8,0]139; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1140; CHECK-NEXT: vpermw %ymm0, %ymm2, %ymm0 {%k1} {z}141; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0142; CHECK-NEXT: vzeroupper143; CHECK-NEXT: retq144 %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 14, i32 15, i32 7, i32 13, i32 4, i32 12, i32 8, i32 0>145 %cmp = icmp eq <8 x i16> %mask, zeroinitializer146 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer147 ret <8 x i16> %res148}149define <8 x i16> @test_16xi16_to_8xi16_perm_mem_mask0(ptr %vp) {150; CHECK-LABEL: test_16xi16_to_8xi16_perm_mem_mask0:151; CHECK: # %bb.0:152; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm0 = [0,7,13,3,5,13,3,9]153; CHECK-NEXT: vpermw (%rdi), %ymm0, %ymm0154; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0155; CHECK-NEXT: vzeroupper156; CHECK-NEXT: retq157 %vec = load <16 x i16>, ptr %vp158 %res = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 0, i32 7, i32 13, i32 3, i32 5, i32 13, i32 3, i32 9>159 ret <8 x i16> %res160}161define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mem_mask0(ptr %vp, <8 x i16> %vec2, <8 x i16> %mask) {162; CHECK-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask0:163; CHECK: # %bb.0:164; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0165; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [0,7,13,3,5,13,3,9]166; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1167; CHECK-NEXT: vpermw (%rdi), %ymm2, %ymm0 {%k1}168; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0169; CHECK-NEXT: vzeroupper170; CHECK-NEXT: retq171 %vec = load <16 x i16>, ptr %vp172 %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 0, i32 7, i32 13, i32 3, i32 5, i32 13, i32 3, i32 9>173 %cmp = icmp eq <8 x i16> %mask, zeroinitializer174 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2175 ret <8 x i16> %res176}177 178define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mem_mask0(ptr %vp, <8 x i16> %mask) {179; CHECK-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask0:180; CHECK: # %bb.0:181; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm1 = [0,7,13,3,5,13,3,9]182; CHECK-NEXT: vptestnmw %xmm0, %xmm0, %k1183; CHECK-NEXT: vpermw (%rdi), %ymm1, %ymm0 {%k1} {z}184; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0185; CHECK-NEXT: vzeroupper186; CHECK-NEXT: retq187 %vec = load <16 x i16>, ptr %vp188 %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 0, i32 7, i32 13, i32 3, i32 5, i32 13, i32 3, i32 9>189 %cmp = icmp eq <8 x i16> %mask, zeroinitializer190 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer191 ret <8 x i16> %res192}193 194define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mem_mask1(ptr %vp, <8 x i16> %vec2, <8 x i16> %mask) {195; CHECK-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask1:196; CHECK: # %bb.0:197; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0198; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [3,15,12,7,1,5,8,14]199; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1200; CHECK-NEXT: vpermw (%rdi), %ymm2, %ymm0 {%k1}201; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0202; CHECK-NEXT: vzeroupper203; CHECK-NEXT: retq204 %vec = load <16 x i16>, ptr %vp205 %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 3, i32 15, i32 12, i32 7, i32 1, i32 5, i32 8, i32 14>206 %cmp = icmp eq <8 x i16> %mask, zeroinitializer207 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2208 ret <8 x i16> %res209}210 211define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mem_mask1(ptr %vp, <8 x i16> %mask) {212; CHECK-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask1:213; CHECK: # %bb.0:214; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm1 = [3,15,12,7,1,5,8,14]215; CHECK-NEXT: vptestnmw %xmm0, %xmm0, %k1216; CHECK-NEXT: vpermw (%rdi), %ymm1, %ymm0 {%k1} {z}217; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0218; CHECK-NEXT: vzeroupper219; CHECK-NEXT: retq220 %vec = load <16 x i16>, ptr %vp221 %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 3, i32 15, i32 12, i32 7, i32 1, i32 5, i32 8, i32 14>222 %cmp = icmp eq <8 x i16> %mask, zeroinitializer223 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer224 ret <8 x i16> %res225}226 227define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mem_mask2(ptr %vp, <8 x i16> %vec2, <8 x i16> %mask) {228; CHECK-FAST-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask2:229; CHECK-FAST: # %bb.0:230; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0231; CHECK-FAST-NEXT: vpmovsxbw {{.*#+}} xmm2 = [1,8,11,8,13,8,15,9]232; CHECK-FAST-NEXT: vptestnmw %xmm1, %xmm1, %k1233; CHECK-FAST-NEXT: vpermw (%rdi), %ymm2, %ymm0 {%k1}234; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0235; CHECK-FAST-NEXT: vzeroupper236; CHECK-FAST-NEXT: retq237;238; CHECK-FAST-PERLANE-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask2:239; CHECK-FAST-PERLANE: # %bb.0:240; CHECK-FAST-PERLANE-NEXT: vpsrld $16, (%rdi), %xmm2241; CHECK-FAST-PERLANE-NEXT: vmovdqa 16(%rdi), %xmm3242; CHECK-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,u,0,1,6,7,0,1,10,11,0,1,14,15,2,3]243; CHECK-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3,4,5,6,7]244; CHECK-FAST-PERLANE-NEXT: vptestnmw %xmm1, %xmm1, %k1245; CHECK-FAST-PERLANE-NEXT: vmovdqu16 %xmm2, %xmm0 {%k1}246; CHECK-FAST-PERLANE-NEXT: retq247 %vec = load <16 x i16>, ptr %vp248 %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 1, i32 8, i32 11, i32 8, i32 13, i32 8, i32 15, i32 9>249 %cmp = icmp eq <8 x i16> %mask, zeroinitializer250 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2251 ret <8 x i16> %res252}253 254define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mem_mask2(ptr %vp, <8 x i16> %mask) {255; CHECK-FAST-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask2:256; CHECK-FAST: # %bb.0:257; CHECK-FAST-NEXT: vpmovsxbw {{.*#+}} xmm1 = [1,8,11,8,13,8,15,9]258; CHECK-FAST-NEXT: vptestnmw %xmm0, %xmm0, %k1259; CHECK-FAST-NEXT: vpermw (%rdi), %ymm1, %ymm0 {%k1} {z}260; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0261; CHECK-FAST-NEXT: vzeroupper262; CHECK-FAST-NEXT: retq263;264; CHECK-FAST-PERLANE-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask2:265; CHECK-FAST-PERLANE: # %bb.0:266; CHECK-FAST-PERLANE-NEXT: vpsrld $16, (%rdi), %xmm1267; CHECK-FAST-PERLANE-NEXT: vmovdqa 16(%rdi), %xmm2268; CHECK-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[u,u,0,1,6,7,0,1,10,11,0,1,14,15,2,3]269; CHECK-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]270; CHECK-FAST-PERLANE-NEXT: vptestnmw %xmm0, %xmm0, %k1271; CHECK-FAST-PERLANE-NEXT: vmovdqu16 %xmm1, %xmm0 {%k1} {z}272; CHECK-FAST-PERLANE-NEXT: retq273 %vec = load <16 x i16>, ptr %vp274 %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 1, i32 8, i32 11, i32 8, i32 13, i32 8, i32 15, i32 9>275 %cmp = icmp eq <8 x i16> %mask, zeroinitializer276 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer277 ret <8 x i16> %res278}279 280define <8 x i16> @test_16xi16_to_8xi16_perm_mem_mask3(ptr %vp) {281; CHECK-LABEL: test_16xi16_to_8xi16_perm_mem_mask3:282; CHECK: # %bb.0:283; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm0 = [9,7,9,6,9,4,3,2]284; CHECK-NEXT: vpermw (%rdi), %ymm0, %ymm0285; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0286; CHECK-NEXT: vzeroupper287; CHECK-NEXT: retq288 %vec = load <16 x i16>, ptr %vp289 %res = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 9, i32 7, i32 9, i32 6, i32 9, i32 4, i32 3, i32 2>290 ret <8 x i16> %res291}292define <8 x i16> @test_masked_16xi16_to_8xi16_perm_mem_mask3(ptr %vp, <8 x i16> %vec2, <8 x i16> %mask) {293; CHECK-LABEL: test_masked_16xi16_to_8xi16_perm_mem_mask3:294; CHECK: # %bb.0:295; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0296; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [9,7,9,6,9,4,3,2]297; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1298; CHECK-NEXT: vpermw (%rdi), %ymm2, %ymm0 {%k1}299; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0300; CHECK-NEXT: vzeroupper301; CHECK-NEXT: retq302 %vec = load <16 x i16>, ptr %vp303 %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 9, i32 7, i32 9, i32 6, i32 9, i32 4, i32 3, i32 2>304 %cmp = icmp eq <8 x i16> %mask, zeroinitializer305 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2306 ret <8 x i16> %res307}308 309define <8 x i16> @test_masked_z_16xi16_to_8xi16_perm_mem_mask3(ptr %vp, <8 x i16> %mask) {310; CHECK-LABEL: test_masked_z_16xi16_to_8xi16_perm_mem_mask3:311; CHECK: # %bb.0:312; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm1 = [9,7,9,6,9,4,3,2]313; CHECK-NEXT: vptestnmw %xmm0, %xmm0, %k1314; CHECK-NEXT: vpermw (%rdi), %ymm1, %ymm0 {%k1} {z}315; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0316; CHECK-NEXT: vzeroupper317; CHECK-NEXT: retq318 %vec = load <16 x i16>, ptr %vp319 %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 9, i32 7, i32 9, i32 6, i32 9, i32 4, i32 3, i32 2>320 %cmp = icmp eq <8 x i16> %mask, zeroinitializer321 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer322 ret <8 x i16> %res323}324 325define <16 x i16> @test_32xi16_to_16xi16_perm_mask0(<32 x i16> %vec) {326; CHECK-LABEL: test_32xi16_to_16xi16_perm_mask0:327; CHECK: # %bb.0:328; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm1 = [24,28,29,26,28,29,17,12,22,8,25,27,28,18,30,18]329; CHECK-NEXT: vpermw %zmm0, %zmm1, %zmm0330; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0331; CHECK-NEXT: retq332 %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 24, i32 28, i32 29, i32 26, i32 28, i32 29, i32 17, i32 12, i32 22, i32 8, i32 25, i32 27, i32 28, i32 18, i32 30, i32 18>333 ret <16 x i16> %res334}335define <16 x i16> @test_masked_32xi16_to_16xi16_perm_mask0(<32 x i16> %vec, <16 x i16> %vec2, <16 x i16> %mask) {336; CHECK-LABEL: test_masked_32xi16_to_16xi16_perm_mask0:337; CHECK: # %bb.0:338; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1339; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm3 = [24,28,29,26,28,29,17,12,22,8,25,27,28,18,30,18]340; CHECK-NEXT: vptestnmw %ymm2, %ymm2, %k1341; CHECK-NEXT: vpermw %zmm0, %zmm3, %zmm1 {%k1}342; CHECK-NEXT: vmovdqa %ymm1, %ymm0343; CHECK-NEXT: retq344 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 24, i32 28, i32 29, i32 26, i32 28, i32 29, i32 17, i32 12, i32 22, i32 8, i32 25, i32 27, i32 28, i32 18, i32 30, i32 18>345 %cmp = icmp eq <16 x i16> %mask, zeroinitializer346 %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2347 ret <16 x i16> %res348}349 350define <16 x i16> @test_masked_z_32xi16_to_16xi16_perm_mask0(<32 x i16> %vec, <16 x i16> %mask) {351; CHECK-LABEL: test_masked_z_32xi16_to_16xi16_perm_mask0:352; CHECK: # %bb.0:353; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm2 = [24,28,29,26,28,29,17,12,22,8,25,27,28,18,30,18]354; CHECK-NEXT: vptestnmw %ymm1, %ymm1, %k1355; CHECK-NEXT: vpermw %zmm0, %zmm2, %zmm0 {%k1} {z}356; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0357; CHECK-NEXT: retq358 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 24, i32 28, i32 29, i32 26, i32 28, i32 29, i32 17, i32 12, i32 22, i32 8, i32 25, i32 27, i32 28, i32 18, i32 30, i32 18>359 %cmp = icmp eq <16 x i16> %mask, zeroinitializer360 %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer361 ret <16 x i16> %res362}363define <16 x i16> @test_masked_32xi16_to_16xi16_perm_mask1(<32 x i16> %vec, <16 x i16> %vec2, <16 x i16> %mask) {364; CHECK-LABEL: test_masked_32xi16_to_16xi16_perm_mask1:365; CHECK: # %bb.0:366; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1367; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm3 = [14,21,31,29,25,2,19,15,20,27,7,23,3,7,25,10]368; CHECK-NEXT: vptestnmw %ymm2, %ymm2, %k1369; CHECK-NEXT: vpermw %zmm0, %zmm3, %zmm1 {%k1}370; CHECK-NEXT: vmovdqa %ymm1, %ymm0371; CHECK-NEXT: retq372 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 14, i32 21, i32 31, i32 29, i32 25, i32 2, i32 19, i32 15, i32 20, i32 27, i32 7, i32 23, i32 3, i32 7, i32 25, i32 10>373 %cmp = icmp eq <16 x i16> %mask, zeroinitializer374 %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2375 ret <16 x i16> %res376}377 378define <16 x i16> @test_masked_z_32xi16_to_16xi16_perm_mask1(<32 x i16> %vec, <16 x i16> %mask) {379; CHECK-LABEL: test_masked_z_32xi16_to_16xi16_perm_mask1:380; CHECK: # %bb.0:381; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm2 = [14,21,31,29,25,2,19,15,20,27,7,23,3,7,25,10]382; CHECK-NEXT: vptestnmw %ymm1, %ymm1, %k1383; CHECK-NEXT: vpermw %zmm0, %zmm2, %zmm0 {%k1} {z}384; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0385; CHECK-NEXT: retq386 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 14, i32 21, i32 31, i32 29, i32 25, i32 2, i32 19, i32 15, i32 20, i32 27, i32 7, i32 23, i32 3, i32 7, i32 25, i32 10>387 %cmp = icmp eq <16 x i16> %mask, zeroinitializer388 %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer389 ret <16 x i16> %res390}391define <16 x i16> @test_masked_32xi16_to_16xi16_perm_mask2(<32 x i16> %vec, <16 x i16> %vec2, <16 x i16> %mask) {392; CHECK-LABEL: test_masked_32xi16_to_16xi16_perm_mask2:393; CHECK: # %bb.0:394; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1395; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm3 = [26,3,4,22,1,18,29,17,21,0,20,19,18,12,11,31]396; CHECK-NEXT: vptestnmw %ymm2, %ymm2, %k1397; CHECK-NEXT: vpermw %zmm0, %zmm3, %zmm1 {%k1}398; CHECK-NEXT: vmovdqa %ymm1, %ymm0399; CHECK-NEXT: retq400 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 26, i32 3, i32 4, i32 22, i32 1, i32 18, i32 29, i32 17, i32 21, i32 0, i32 20, i32 19, i32 18, i32 12, i32 11, i32 31>401 %cmp = icmp eq <16 x i16> %mask, zeroinitializer402 %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2403 ret <16 x i16> %res404}405 406define <16 x i16> @test_masked_z_32xi16_to_16xi16_perm_mask2(<32 x i16> %vec, <16 x i16> %mask) {407; CHECK-LABEL: test_masked_z_32xi16_to_16xi16_perm_mask2:408; CHECK: # %bb.0:409; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm2 = [26,3,4,22,1,18,29,17,21,0,20,19,18,12,11,31]410; CHECK-NEXT: vptestnmw %ymm1, %ymm1, %k1411; CHECK-NEXT: vpermw %zmm0, %zmm2, %zmm0 {%k1} {z}412; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0413; CHECK-NEXT: retq414 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 26, i32 3, i32 4, i32 22, i32 1, i32 18, i32 29, i32 17, i32 21, i32 0, i32 20, i32 19, i32 18, i32 12, i32 11, i32 31>415 %cmp = icmp eq <16 x i16> %mask, zeroinitializer416 %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer417 ret <16 x i16> %res418}419define <16 x i16> @test_32xi16_to_16xi16_perm_mask3(<32 x i16> %vec) {420; CHECK-LABEL: test_32xi16_to_16xi16_perm_mask3:421; CHECK: # %bb.0:422; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm1 = [1,0,30,5,3,6,25,29,0,13,3,8,7,20,11,5]423; CHECK-NEXT: vpermw %zmm0, %zmm1, %zmm0424; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0425; CHECK-NEXT: retq426 %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 1, i32 0, i32 30, i32 5, i32 3, i32 6, i32 25, i32 29, i32 0, i32 13, i32 3, i32 8, i32 7, i32 20, i32 11, i32 5>427 ret <16 x i16> %res428}429define <16 x i16> @test_masked_32xi16_to_16xi16_perm_mask3(<32 x i16> %vec, <16 x i16> %vec2, <16 x i16> %mask) {430; CHECK-LABEL: test_masked_32xi16_to_16xi16_perm_mask3:431; CHECK: # %bb.0:432; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1433; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm3 = [1,0,30,5,3,6,25,29,0,13,3,8,7,20,11,5]434; CHECK-NEXT: vptestnmw %ymm2, %ymm2, %k1435; CHECK-NEXT: vpermw %zmm0, %zmm3, %zmm1 {%k1}436; CHECK-NEXT: vmovdqa %ymm1, %ymm0437; CHECK-NEXT: retq438 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 1, i32 0, i32 30, i32 5, i32 3, i32 6, i32 25, i32 29, i32 0, i32 13, i32 3, i32 8, i32 7, i32 20, i32 11, i32 5>439 %cmp = icmp eq <16 x i16> %mask, zeroinitializer440 %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2441 ret <16 x i16> %res442}443 444define <16 x i16> @test_masked_z_32xi16_to_16xi16_perm_mask3(<32 x i16> %vec, <16 x i16> %mask) {445; CHECK-LABEL: test_masked_z_32xi16_to_16xi16_perm_mask3:446; CHECK: # %bb.0:447; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm2 = [1,0,30,5,3,6,25,29,0,13,3,8,7,20,11,5]448; CHECK-NEXT: vptestnmw %ymm1, %ymm1, %k1449; CHECK-NEXT: vpermw %zmm0, %zmm2, %zmm0 {%k1} {z}450; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0451; CHECK-NEXT: retq452 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 1, i32 0, i32 30, i32 5, i32 3, i32 6, i32 25, i32 29, i32 0, i32 13, i32 3, i32 8, i32 7, i32 20, i32 11, i32 5>453 %cmp = icmp eq <16 x i16> %mask, zeroinitializer454 %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer455 ret <16 x i16> %res456}457define <8 x i16> @test_32xi16_to_8xi16_perm_mask0(<32 x i16> %vec) {458; CHECK-LABEL: test_32xi16_to_8xi16_perm_mask0:459; CHECK: # %bb.0:460; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm1 = [6,11,23,26,29,5,21,30]461; CHECK-NEXT: vpermw %zmm0, %zmm1, %zmm0462; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0463; CHECK-NEXT: vzeroupper464; CHECK-NEXT: retq465 %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 6, i32 11, i32 23, i32 26, i32 29, i32 5, i32 21, i32 30>466 ret <8 x i16> %res467}468define <8 x i16> @test_masked_32xi16_to_8xi16_perm_mask0(<32 x i16> %vec, <8 x i16> %vec2, <8 x i16> %mask) {469; CHECK-LABEL: test_masked_32xi16_to_8xi16_perm_mask0:470; CHECK: # %bb.0:471; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1472; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm3 = [6,11,23,26,29,5,21,30]473; CHECK-NEXT: vptestnmw %xmm2, %xmm2, %k1474; CHECK-NEXT: vpermw %zmm0, %zmm3, %zmm1 {%k1}475; CHECK-NEXT: vmovdqa %xmm1, %xmm0476; CHECK-NEXT: vzeroupper477; CHECK-NEXT: retq478 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 6, i32 11, i32 23, i32 26, i32 29, i32 5, i32 21, i32 30>479 %cmp = icmp eq <8 x i16> %mask, zeroinitializer480 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2481 ret <8 x i16> %res482}483 484define <8 x i16> @test_masked_z_32xi16_to_8xi16_perm_mask0(<32 x i16> %vec, <8 x i16> %mask) {485; CHECK-LABEL: test_masked_z_32xi16_to_8xi16_perm_mask0:486; CHECK: # %bb.0:487; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [6,11,23,26,29,5,21,30]488; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1489; CHECK-NEXT: vpermw %zmm0, %zmm2, %zmm0 {%k1} {z}490; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0491; CHECK-NEXT: vzeroupper492; CHECK-NEXT: retq493 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 6, i32 11, i32 23, i32 26, i32 29, i32 5, i32 21, i32 30>494 %cmp = icmp eq <8 x i16> %mask, zeroinitializer495 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer496 ret <8 x i16> %res497}498define <8 x i16> @test_masked_32xi16_to_8xi16_perm_mask1(<32 x i16> %vec, <8 x i16> %vec2, <8 x i16> %mask) {499; CHECK-LABEL: test_masked_32xi16_to_8xi16_perm_mask1:500; CHECK: # %bb.0:501; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1502; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm3 = [1,21,27,10,8,19,14,5]503; CHECK-NEXT: vptestnmw %xmm2, %xmm2, %k1504; CHECK-NEXT: vpermw %zmm0, %zmm3, %zmm1 {%k1}505; CHECK-NEXT: vmovdqa %xmm1, %xmm0506; CHECK-NEXT: vzeroupper507; CHECK-NEXT: retq508 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 1, i32 21, i32 27, i32 10, i32 8, i32 19, i32 14, i32 5>509 %cmp = icmp eq <8 x i16> %mask, zeroinitializer510 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2511 ret <8 x i16> %res512}513 514define <8 x i16> @test_masked_z_32xi16_to_8xi16_perm_mask1(<32 x i16> %vec, <8 x i16> %mask) {515; CHECK-LABEL: test_masked_z_32xi16_to_8xi16_perm_mask1:516; CHECK: # %bb.0:517; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [1,21,27,10,8,19,14,5]518; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1519; CHECK-NEXT: vpermw %zmm0, %zmm2, %zmm0 {%k1} {z}520; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0521; CHECK-NEXT: vzeroupper522; CHECK-NEXT: retq523 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 1, i32 21, i32 27, i32 10, i32 8, i32 19, i32 14, i32 5>524 %cmp = icmp eq <8 x i16> %mask, zeroinitializer525 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer526 ret <8 x i16> %res527}528define <8 x i16> @test_masked_32xi16_to_8xi16_perm_mask2(<32 x i16> %vec, <8 x i16> %vec2, <8 x i16> %mask) {529; CHECK-LABEL: test_masked_32xi16_to_8xi16_perm_mask2:530; CHECK: # %bb.0:531; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1532; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm3 = [15,13,18,16,9,11,26,8]533; CHECK-NEXT: vptestnmw %xmm2, %xmm2, %k1534; CHECK-NEXT: vpermw %zmm0, %zmm3, %zmm1 {%k1}535; CHECK-NEXT: vmovdqa %xmm1, %xmm0536; CHECK-NEXT: vzeroupper537; CHECK-NEXT: retq538 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 15, i32 13, i32 18, i32 16, i32 9, i32 11, i32 26, i32 8>539 %cmp = icmp eq <8 x i16> %mask, zeroinitializer540 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2541 ret <8 x i16> %res542}543 544define <8 x i16> @test_masked_z_32xi16_to_8xi16_perm_mask2(<32 x i16> %vec, <8 x i16> %mask) {545; CHECK-LABEL: test_masked_z_32xi16_to_8xi16_perm_mask2:546; CHECK: # %bb.0:547; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [15,13,18,16,9,11,26,8]548; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1549; CHECK-NEXT: vpermw %zmm0, %zmm2, %zmm0 {%k1} {z}550; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0551; CHECK-NEXT: vzeroupper552; CHECK-NEXT: retq553 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 15, i32 13, i32 18, i32 16, i32 9, i32 11, i32 26, i32 8>554 %cmp = icmp eq <8 x i16> %mask, zeroinitializer555 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer556 ret <8 x i16> %res557}558define <8 x i16> @test_32xi16_to_8xi16_perm_mask3(<32 x i16> %vec) {559; CHECK-LABEL: test_32xi16_to_8xi16_perm_mask3:560; CHECK: # %bb.0:561; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm1 = [17,0,23,10,1,8,7,30]562; CHECK-NEXT: vpermw %zmm0, %zmm1, %zmm0563; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0564; CHECK-NEXT: vzeroupper565; CHECK-NEXT: retq566 %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 17, i32 0, i32 23, i32 10, i32 1, i32 8, i32 7, i32 30>567 ret <8 x i16> %res568}569define <8 x i16> @test_masked_32xi16_to_8xi16_perm_mask3(<32 x i16> %vec, <8 x i16> %vec2, <8 x i16> %mask) {570; CHECK-LABEL: test_masked_32xi16_to_8xi16_perm_mask3:571; CHECK: # %bb.0:572; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1573; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm3 = [17,0,23,10,1,8,7,30]574; CHECK-NEXT: vptestnmw %xmm2, %xmm2, %k1575; CHECK-NEXT: vpermw %zmm0, %zmm3, %zmm1 {%k1}576; CHECK-NEXT: vmovdqa %xmm1, %xmm0577; CHECK-NEXT: vzeroupper578; CHECK-NEXT: retq579 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 17, i32 0, i32 23, i32 10, i32 1, i32 8, i32 7, i32 30>580 %cmp = icmp eq <8 x i16> %mask, zeroinitializer581 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2582 ret <8 x i16> %res583}584 585define <8 x i16> @test_masked_z_32xi16_to_8xi16_perm_mask3(<32 x i16> %vec, <8 x i16> %mask) {586; CHECK-LABEL: test_masked_z_32xi16_to_8xi16_perm_mask3:587; CHECK: # %bb.0:588; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [17,0,23,10,1,8,7,30]589; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1590; CHECK-NEXT: vpermw %zmm0, %zmm2, %zmm0 {%k1} {z}591; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0592; CHECK-NEXT: vzeroupper593; CHECK-NEXT: retq594 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 17, i32 0, i32 23, i32 10, i32 1, i32 8, i32 7, i32 30>595 %cmp = icmp eq <8 x i16> %mask, zeroinitializer596 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer597 ret <8 x i16> %res598}599define <16 x i16> @test_32xi16_to_16xi16_perm_mem_mask0(ptr %vp) {600; CHECK-LABEL: test_32xi16_to_16xi16_perm_mem_mask0:601; CHECK: # %bb.0:602; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm0 = [20,19,22,12,13,20,0,6,10,7,20,12,28,18,13,12]603; CHECK-NEXT: vpermw (%rdi), %zmm0, %zmm0604; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0605; CHECK-NEXT: retq606 %vec = load <32 x i16>, ptr %vp607 %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 20, i32 19, i32 22, i32 12, i32 13, i32 20, i32 0, i32 6, i32 10, i32 7, i32 20, i32 12, i32 28, i32 18, i32 13, i32 12>608 ret <16 x i16> %res609}610define <16 x i16> @test_masked_32xi16_to_16xi16_perm_mem_mask0(ptr %vp, <16 x i16> %vec2, <16 x i16> %mask) {611; CHECK-LABEL: test_masked_32xi16_to_16xi16_perm_mem_mask0:612; CHECK: # %bb.0:613; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0614; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm2 = [20,19,22,12,13,20,0,6,10,7,20,12,28,18,13,12]615; CHECK-NEXT: vptestnmw %ymm1, %ymm1, %k1616; CHECK-NEXT: vpermw (%rdi), %zmm2, %zmm0 {%k1}617; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0618; CHECK-NEXT: retq619 %vec = load <32 x i16>, ptr %vp620 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 20, i32 19, i32 22, i32 12, i32 13, i32 20, i32 0, i32 6, i32 10, i32 7, i32 20, i32 12, i32 28, i32 18, i32 13, i32 12>621 %cmp = icmp eq <16 x i16> %mask, zeroinitializer622 %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2623 ret <16 x i16> %res624}625 626define <16 x i16> @test_masked_z_32xi16_to_16xi16_perm_mem_mask0(ptr %vp, <16 x i16> %mask) {627; CHECK-LABEL: test_masked_z_32xi16_to_16xi16_perm_mem_mask0:628; CHECK: # %bb.0:629; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm1 = [20,19,22,12,13,20,0,6,10,7,20,12,28,18,13,12]630; CHECK-NEXT: vptestnmw %ymm0, %ymm0, %k1631; CHECK-NEXT: vpermw (%rdi), %zmm1, %zmm0 {%k1} {z}632; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0633; CHECK-NEXT: retq634 %vec = load <32 x i16>, ptr %vp635 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 20, i32 19, i32 22, i32 12, i32 13, i32 20, i32 0, i32 6, i32 10, i32 7, i32 20, i32 12, i32 28, i32 18, i32 13, i32 12>636 %cmp = icmp eq <16 x i16> %mask, zeroinitializer637 %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer638 ret <16 x i16> %res639}640 641define <16 x i16> @test_masked_32xi16_to_16xi16_perm_mem_mask1(ptr %vp, <16 x i16> %vec2, <16 x i16> %mask) {642; CHECK-LABEL: test_masked_32xi16_to_16xi16_perm_mem_mask1:643; CHECK: # %bb.0:644; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0645; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm2 = [22,13,21,1,14,8,5,16,15,17,24,28,15,9,14,25]646; CHECK-NEXT: vptestnmw %ymm1, %ymm1, %k1647; CHECK-NEXT: vpermw (%rdi), %zmm2, %zmm0 {%k1}648; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0649; CHECK-NEXT: retq650 %vec = load <32 x i16>, ptr %vp651 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 22, i32 13, i32 21, i32 1, i32 14, i32 8, i32 5, i32 16, i32 15, i32 17, i32 24, i32 28, i32 15, i32 9, i32 14, i32 25>652 %cmp = icmp eq <16 x i16> %mask, zeroinitializer653 %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2654 ret <16 x i16> %res655}656 657define <16 x i16> @test_masked_z_32xi16_to_16xi16_perm_mem_mask1(ptr %vp, <16 x i16> %mask) {658; CHECK-LABEL: test_masked_z_32xi16_to_16xi16_perm_mem_mask1:659; CHECK: # %bb.0:660; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm1 = [22,13,21,1,14,8,5,16,15,17,24,28,15,9,14,25]661; CHECK-NEXT: vptestnmw %ymm0, %ymm0, %k1662; CHECK-NEXT: vpermw (%rdi), %zmm1, %zmm0 {%k1} {z}663; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0664; CHECK-NEXT: retq665 %vec = load <32 x i16>, ptr %vp666 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 22, i32 13, i32 21, i32 1, i32 14, i32 8, i32 5, i32 16, i32 15, i32 17, i32 24, i32 28, i32 15, i32 9, i32 14, i32 25>667 %cmp = icmp eq <16 x i16> %mask, zeroinitializer668 %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer669 ret <16 x i16> %res670}671 672define <16 x i16> @test_masked_32xi16_to_16xi16_perm_mem_mask2(ptr %vp, <16 x i16> %vec2, <16 x i16> %mask) {673; CHECK-LABEL: test_masked_32xi16_to_16xi16_perm_mem_mask2:674; CHECK: # %bb.0:675; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0676; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm2 = [28,25,6,31,20,2,23,31,12,21,10,6,22,0,26,16]677; CHECK-NEXT: vptestnmw %ymm1, %ymm1, %k1678; CHECK-NEXT: vpermw (%rdi), %zmm2, %zmm0 {%k1}679; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0680; CHECK-NEXT: retq681 %vec = load <32 x i16>, ptr %vp682 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 28, i32 25, i32 6, i32 31, i32 20, i32 2, i32 23, i32 31, i32 12, i32 21, i32 10, i32 6, i32 22, i32 0, i32 26, i32 16>683 %cmp = icmp eq <16 x i16> %mask, zeroinitializer684 %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2685 ret <16 x i16> %res686}687 688define <16 x i16> @test_masked_z_32xi16_to_16xi16_perm_mem_mask2(ptr %vp, <16 x i16> %mask) {689; CHECK-LABEL: test_masked_z_32xi16_to_16xi16_perm_mem_mask2:690; CHECK: # %bb.0:691; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm1 = [28,25,6,31,20,2,23,31,12,21,10,6,22,0,26,16]692; CHECK-NEXT: vptestnmw %ymm0, %ymm0, %k1693; CHECK-NEXT: vpermw (%rdi), %zmm1, %zmm0 {%k1} {z}694; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0695; CHECK-NEXT: retq696 %vec = load <32 x i16>, ptr %vp697 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 28, i32 25, i32 6, i32 31, i32 20, i32 2, i32 23, i32 31, i32 12, i32 21, i32 10, i32 6, i32 22, i32 0, i32 26, i32 16>698 %cmp = icmp eq <16 x i16> %mask, zeroinitializer699 %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer700 ret <16 x i16> %res701}702 703define <16 x i16> @test_32xi16_to_16xi16_perm_mem_mask3(ptr %vp) {704; CHECK-LABEL: test_32xi16_to_16xi16_perm_mem_mask3:705; CHECK: # %bb.0:706; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm0 = [3,3,20,27,8,31,3,27,12,2,8,14,25,27,4,16]707; CHECK-NEXT: vpermw (%rdi), %zmm0, %zmm0708; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0709; CHECK-NEXT: retq710 %vec = load <32 x i16>, ptr %vp711 %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 3, i32 3, i32 20, i32 27, i32 8, i32 31, i32 3, i32 27, i32 12, i32 2, i32 8, i32 14, i32 25, i32 27, i32 4, i32 16>712 ret <16 x i16> %res713}714define <16 x i16> @test_masked_32xi16_to_16xi16_perm_mem_mask3(ptr %vp, <16 x i16> %vec2, <16 x i16> %mask) {715; CHECK-LABEL: test_masked_32xi16_to_16xi16_perm_mem_mask3:716; CHECK: # %bb.0:717; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0718; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm2 = [3,3,20,27,8,31,3,27,12,2,8,14,25,27,4,16]719; CHECK-NEXT: vptestnmw %ymm1, %ymm1, %k1720; CHECK-NEXT: vpermw (%rdi), %zmm2, %zmm0 {%k1}721; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0722; CHECK-NEXT: retq723 %vec = load <32 x i16>, ptr %vp724 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 3, i32 3, i32 20, i32 27, i32 8, i32 31, i32 3, i32 27, i32 12, i32 2, i32 8, i32 14, i32 25, i32 27, i32 4, i32 16>725 %cmp = icmp eq <16 x i16> %mask, zeroinitializer726 %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2727 ret <16 x i16> %res728}729 730define <16 x i16> @test_masked_z_32xi16_to_16xi16_perm_mem_mask3(ptr %vp, <16 x i16> %mask) {731; CHECK-LABEL: test_masked_z_32xi16_to_16xi16_perm_mem_mask3:732; CHECK: # %bb.0:733; CHECK-NEXT: vpmovsxbw {{.*#+}} ymm1 = [3,3,20,27,8,31,3,27,12,2,8,14,25,27,4,16]734; CHECK-NEXT: vptestnmw %ymm0, %ymm0, %k1735; CHECK-NEXT: vpermw (%rdi), %zmm1, %zmm0 {%k1} {z}736; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0737; CHECK-NEXT: retq738 %vec = load <32 x i16>, ptr %vp739 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <16 x i32> <i32 3, i32 3, i32 20, i32 27, i32 8, i32 31, i32 3, i32 27, i32 12, i32 2, i32 8, i32 14, i32 25, i32 27, i32 4, i32 16>740 %cmp = icmp eq <16 x i16> %mask, zeroinitializer741 %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer742 ret <16 x i16> %res743}744 745define <8 x i16> @test_32xi16_to_8xi16_perm_mem_mask0(ptr %vp) {746; CHECK-LABEL: test_32xi16_to_8xi16_perm_mem_mask0:747; CHECK: # %bb.0:748; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm0 = [0,1,21,17,30,30,29,1]749; CHECK-NEXT: vpermw (%rdi), %zmm0, %zmm0750; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0751; CHECK-NEXT: vzeroupper752; CHECK-NEXT: retq753 %vec = load <32 x i16>, ptr %vp754 %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 0, i32 1, i32 21, i32 17, i32 30, i32 30, i32 29, i32 1>755 ret <8 x i16> %res756}757define <8 x i16> @test_masked_32xi16_to_8xi16_perm_mem_mask0(ptr %vp, <8 x i16> %vec2, <8 x i16> %mask) {758; CHECK-LABEL: test_masked_32xi16_to_8xi16_perm_mem_mask0:759; CHECK: # %bb.0:760; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0761; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [0,1,21,17,30,30,29,1]762; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1763; CHECK-NEXT: vpermw (%rdi), %zmm2, %zmm0 {%k1}764; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0765; CHECK-NEXT: vzeroupper766; CHECK-NEXT: retq767 %vec = load <32 x i16>, ptr %vp768 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 0, i32 1, i32 21, i32 17, i32 30, i32 30, i32 29, i32 1>769 %cmp = icmp eq <8 x i16> %mask, zeroinitializer770 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2771 ret <8 x i16> %res772}773 774define <8 x i16> @test_masked_z_32xi16_to_8xi16_perm_mem_mask0(ptr %vp, <8 x i16> %mask) {775; CHECK-LABEL: test_masked_z_32xi16_to_8xi16_perm_mem_mask0:776; CHECK: # %bb.0:777; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm1 = [0,1,21,17,30,30,29,1]778; CHECK-NEXT: vptestnmw %xmm0, %xmm0, %k1779; CHECK-NEXT: vpermw (%rdi), %zmm1, %zmm0 {%k1} {z}780; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0781; CHECK-NEXT: vzeroupper782; CHECK-NEXT: retq783 %vec = load <32 x i16>, ptr %vp784 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 0, i32 1, i32 21, i32 17, i32 30, i32 30, i32 29, i32 1>785 %cmp = icmp eq <8 x i16> %mask, zeroinitializer786 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer787 ret <8 x i16> %res788}789 790define <8 x i16> @test_masked_32xi16_to_8xi16_perm_mem_mask1(ptr %vp, <8 x i16> %vec2, <8 x i16> %mask) {791; CHECK-LABEL: test_masked_32xi16_to_8xi16_perm_mem_mask1:792; CHECK: # %bb.0:793; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0794; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [23,22,20,22,28,20,11,17]795; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1796; CHECK-NEXT: vpermw (%rdi), %zmm2, %zmm0 {%k1}797; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0798; CHECK-NEXT: vzeroupper799; CHECK-NEXT: retq800 %vec = load <32 x i16>, ptr %vp801 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 23, i32 22, i32 20, i32 22, i32 28, i32 20, i32 11, i32 17>802 %cmp = icmp eq <8 x i16> %mask, zeroinitializer803 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2804 ret <8 x i16> %res805}806 807define <8 x i16> @test_masked_z_32xi16_to_8xi16_perm_mem_mask1(ptr %vp, <8 x i16> %mask) {808; CHECK-LABEL: test_masked_z_32xi16_to_8xi16_perm_mem_mask1:809; CHECK: # %bb.0:810; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm1 = [23,22,20,22,28,20,11,17]811; CHECK-NEXT: vptestnmw %xmm0, %xmm0, %k1812; CHECK-NEXT: vpermw (%rdi), %zmm1, %zmm0 {%k1} {z}813; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0814; CHECK-NEXT: vzeroupper815; CHECK-NEXT: retq816 %vec = load <32 x i16>, ptr %vp817 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 23, i32 22, i32 20, i32 22, i32 28, i32 20, i32 11, i32 17>818 %cmp = icmp eq <8 x i16> %mask, zeroinitializer819 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer820 ret <8 x i16> %res821}822 823define <8 x i16> @test_masked_32xi16_to_8xi16_perm_mem_mask2(ptr %vp, <8 x i16> %vec2, <8 x i16> %mask) {824; CHECK-LABEL: test_masked_32xi16_to_8xi16_perm_mem_mask2:825; CHECK: # %bb.0:826; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0827; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [6,18,0,4,10,25,22,10]828; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1829; CHECK-NEXT: vpermw (%rdi), %zmm2, %zmm0 {%k1}830; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0831; CHECK-NEXT: vzeroupper832; CHECK-NEXT: retq833 %vec = load <32 x i16>, ptr %vp834 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 6, i32 18, i32 0, i32 4, i32 10, i32 25, i32 22, i32 10>835 %cmp = icmp eq <8 x i16> %mask, zeroinitializer836 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2837 ret <8 x i16> %res838}839 840define <8 x i16> @test_masked_z_32xi16_to_8xi16_perm_mem_mask2(ptr %vp, <8 x i16> %mask) {841; CHECK-LABEL: test_masked_z_32xi16_to_8xi16_perm_mem_mask2:842; CHECK: # %bb.0:843; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm1 = [6,18,0,4,10,25,22,10]844; CHECK-NEXT: vptestnmw %xmm0, %xmm0, %k1845; CHECK-NEXT: vpermw (%rdi), %zmm1, %zmm0 {%k1} {z}846; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0847; CHECK-NEXT: vzeroupper848; CHECK-NEXT: retq849 %vec = load <32 x i16>, ptr %vp850 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 6, i32 18, i32 0, i32 4, i32 10, i32 25, i32 22, i32 10>851 %cmp = icmp eq <8 x i16> %mask, zeroinitializer852 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer853 ret <8 x i16> %res854}855 856define <8 x i16> @test_32xi16_to_8xi16_perm_mem_mask3(ptr %vp) {857; CHECK-LABEL: test_32xi16_to_8xi16_perm_mem_mask3:858; CHECK: # %bb.0:859; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm0 = [19,1,5,31,9,12,17,9]860; CHECK-NEXT: vpermw (%rdi), %zmm0, %zmm0861; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0862; CHECK-NEXT: vzeroupper863; CHECK-NEXT: retq864 %vec = load <32 x i16>, ptr %vp865 %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 19, i32 1, i32 5, i32 31, i32 9, i32 12, i32 17, i32 9>866 ret <8 x i16> %res867}868define <8 x i16> @test_masked_32xi16_to_8xi16_perm_mem_mask3(ptr %vp, <8 x i16> %vec2, <8 x i16> %mask) {869; CHECK-LABEL: test_masked_32xi16_to_8xi16_perm_mem_mask3:870; CHECK: # %bb.0:871; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0872; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm2 = [19,1,5,31,9,12,17,9]873; CHECK-NEXT: vptestnmw %xmm1, %xmm1, %k1874; CHECK-NEXT: vpermw (%rdi), %zmm2, %zmm0 {%k1}875; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0876; CHECK-NEXT: vzeroupper877; CHECK-NEXT: retq878 %vec = load <32 x i16>, ptr %vp879 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 19, i32 1, i32 5, i32 31, i32 9, i32 12, i32 17, i32 9>880 %cmp = icmp eq <8 x i16> %mask, zeroinitializer881 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2882 ret <8 x i16> %res883}884 885define <8 x i16> @test_masked_z_32xi16_to_8xi16_perm_mem_mask3(ptr %vp, <8 x i16> %mask) {886; CHECK-LABEL: test_masked_z_32xi16_to_8xi16_perm_mem_mask3:887; CHECK: # %bb.0:888; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm1 = [19,1,5,31,9,12,17,9]889; CHECK-NEXT: vptestnmw %xmm0, %xmm0, %k1890; CHECK-NEXT: vpermw (%rdi), %zmm1, %zmm0 {%k1} {z}891; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0892; CHECK-NEXT: vzeroupper893; CHECK-NEXT: retq894 %vec = load <32 x i16>, ptr %vp895 %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <8 x i32> <i32 19, i32 1, i32 5, i32 31, i32 9, i32 12, i32 17, i32 9>896 %cmp = icmp eq <8 x i16> %mask, zeroinitializer897 %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer898 ret <8 x i16> %res899}900 901define <8 x i16> @test_16xi16_to_8xi16_E84C94EF(<16 x i16> %vec) {902; CHECK-LABEL: test_16xi16_to_8xi16_E84C94EF:903; CHECK: # %bb.0:904; CHECK-NEXT: vpmovsxbw {{.*#+}} xmm1 = [14,8,4,12,9,4,14,15]905; CHECK-NEXT: vpermw %ymm0, %ymm1, %ymm0906; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0907; CHECK-NEXT: vzeroupper908; CHECK-NEXT: retq909 %res = shufflevector <16 x i16> %vec, <16 x i16> undef, <8 x i32> <i32 14, i32 8, i32 4, i32 12, i32 9, i32 4, i32 14, i32 15>910 ret <8 x i16> %res911}912 913define <4 x i32> @test_8xi32_to_4xi32_perm_mask0(<8 x i32> %vec) {914; CHECK-LABEL: test_8xi32_to_4xi32_perm_mask0:915; CHECK: # %bb.0:916; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [4,0,3,2]917; CHECK-NEXT: vpermps %ymm0, %ymm1, %ymm0918; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0919; CHECK-NEXT: vzeroupper920; CHECK-NEXT: retq921 %res = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 4, i32 0, i32 3, i32 2>922 ret <4 x i32> %res923}924define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mask0(<8 x i32> %vec, <4 x i32> %vec2, <4 x i32> %mask) {925; CHECK-LABEL: test_masked_8xi32_to_4xi32_perm_mask0:926; CHECK: # %bb.0:927; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm1928; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm3 = [4,0,3,2]929; CHECK-NEXT: vptestnmd %xmm2, %xmm2, %k1930; CHECK-NEXT: vpermd %ymm0, %ymm3, %ymm1 {%k1}931; CHECK-NEXT: vmovdqa %xmm1, %xmm0932; CHECK-NEXT: vzeroupper933; CHECK-NEXT: retq934 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 4, i32 0, i32 3, i32 2>935 %cmp = icmp eq <4 x i32> %mask, zeroinitializer936 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec2937 ret <4 x i32> %res938}939 940define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mask0(<8 x i32> %vec, <4 x i32> %mask) {941; CHECK-LABEL: test_masked_z_8xi32_to_4xi32_perm_mask0:942; CHECK: # %bb.0:943; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [4,0,3,2]944; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k1945; CHECK-NEXT: vpermd %ymm0, %ymm2, %ymm0 {%k1} {z}946; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0947; CHECK-NEXT: vzeroupper948; CHECK-NEXT: retq949 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 4, i32 0, i32 3, i32 2>950 %cmp = icmp eq <4 x i32> %mask, zeroinitializer951 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer952 ret <4 x i32> %res953}954define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mask1(<8 x i32> %vec, <4 x i32> %vec2, <4 x i32> %mask) {955; CHECK-LABEL: test_masked_8xi32_to_4xi32_perm_mask1:956; CHECK: # %bb.0:957; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm1958; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm3 = [3,0,7,3]959; CHECK-NEXT: vptestnmd %xmm2, %xmm2, %k1960; CHECK-NEXT: vpermd %ymm0, %ymm3, %ymm1 {%k1}961; CHECK-NEXT: vmovdqa %xmm1, %xmm0962; CHECK-NEXT: vzeroupper963; CHECK-NEXT: retq964 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 3, i32 0, i32 7, i32 3>965 %cmp = icmp eq <4 x i32> %mask, zeroinitializer966 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec2967 ret <4 x i32> %res968}969 970define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mask1(<8 x i32> %vec, <4 x i32> %mask) {971; CHECK-LABEL: test_masked_z_8xi32_to_4xi32_perm_mask1:972; CHECK: # %bb.0:973; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [3,0,7,3]974; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k1975; CHECK-NEXT: vpermd %ymm0, %ymm2, %ymm0 {%k1} {z}976; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0977; CHECK-NEXT: vzeroupper978; CHECK-NEXT: retq979 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 3, i32 0, i32 7, i32 3>980 %cmp = icmp eq <4 x i32> %mask, zeroinitializer981 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer982 ret <4 x i32> %res983}984define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mask2(<8 x i32> %vec, <4 x i32> %vec2, <4 x i32> %mask) {985; CHECK-LABEL: test_masked_8xi32_to_4xi32_perm_mask2:986; CHECK: # %bb.0:987; CHECK-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,1,2,3]988; CHECK-NEXT: vptestnmd %xmm2, %xmm2, %k1989; CHECK-NEXT: vpblendmd %xmm0, %xmm1, %xmm0 {%k1}990; CHECK-NEXT: vzeroupper991; CHECK-NEXT: retq992 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 6, i32 7, i32 2, i32 3>993 %cmp = icmp eq <4 x i32> %mask, zeroinitializer994 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec2995 ret <4 x i32> %res996}997 998define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mask2(<8 x i32> %vec, <4 x i32> %mask) {999; CHECK-LABEL: test_masked_z_8xi32_to_4xi32_perm_mask2:1000; CHECK: # %bb.0:1001; CHECK-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,1,2,3]1002; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k11003; CHECK-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}1004; CHECK-NEXT: vzeroupper1005; CHECK-NEXT: retq1006 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 6, i32 7, i32 2, i32 3>1007 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1008 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer1009 ret <4 x i32> %res1010}1011define <4 x i32> @test_8xi32_to_4xi32_perm_mask3(<8 x i32> %vec) {1012; CHECK-LABEL: test_8xi32_to_4xi32_perm_mask3:1013; CHECK: # %bb.0:1014; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [5,3,2,5]1015; CHECK-NEXT: vpermps %ymm0, %ymm1, %ymm01016; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm01017; CHECK-NEXT: vzeroupper1018; CHECK-NEXT: retq1019 %res = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 3, i32 2, i32 5>1020 ret <4 x i32> %res1021}1022define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mask3(<8 x i32> %vec, <4 x i32> %vec2, <4 x i32> %mask) {1023; CHECK-LABEL: test_masked_8xi32_to_4xi32_perm_mask3:1024; CHECK: # %bb.0:1025; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm11026; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm3 = [5,3,2,5]1027; CHECK-NEXT: vptestnmd %xmm2, %xmm2, %k11028; CHECK-NEXT: vpermd %ymm0, %ymm3, %ymm1 {%k1}1029; CHECK-NEXT: vmovdqa %xmm1, %xmm01030; CHECK-NEXT: vzeroupper1031; CHECK-NEXT: retq1032 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 3, i32 2, i32 5>1033 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1034 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec21035 ret <4 x i32> %res1036}1037 1038define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mask3(<8 x i32> %vec, <4 x i32> %mask) {1039; CHECK-LABEL: test_masked_z_8xi32_to_4xi32_perm_mask3:1040; CHECK: # %bb.0:1041; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [5,3,2,5]1042; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k11043; CHECK-NEXT: vpermd %ymm0, %ymm2, %ymm0 {%k1} {z}1044; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm01045; CHECK-NEXT: vzeroupper1046; CHECK-NEXT: retq1047 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 3, i32 2, i32 5>1048 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1049 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer1050 ret <4 x i32> %res1051}1052define <4 x i32> @test_8xi32_to_4xi32_perm_mem_mask0(ptr %vp) {1053; CHECK-LABEL: test_8xi32_to_4xi32_perm_mem_mask0:1054; CHECK: # %bb.0:1055; CHECK-NEXT: vmovaps 16(%rdi), %xmm01056; CHECK-NEXT: vshufps $7, (%rdi), %xmm0, %xmm0 # xmm0 = xmm0[3,1],mem[0,0]1057; CHECK-NEXT: retq1058 %vec = load <8 x i32>, ptr %vp1059 %res = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 7, i32 5, i32 0, i32 0>1060 ret <4 x i32> %res1061}1062define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mem_mask0(ptr %vp, <4 x i32> %vec2, <4 x i32> %mask) {1063; CHECK-LABEL: test_masked_8xi32_to_4xi32_perm_mem_mask0:1064; CHECK: # %bb.0:1065; CHECK-NEXT: vmovaps 16(%rdi), %xmm21066; CHECK-NEXT: vshufps $7, (%rdi), %xmm2, %xmm2 # xmm2 = xmm2[3,1],mem[0,0]1067; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k11068; CHECK-NEXT: vmovdqa32 %xmm2, %xmm0 {%k1}1069; CHECK-NEXT: retq1070 %vec = load <8 x i32>, ptr %vp1071 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 7, i32 5, i32 0, i32 0>1072 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1073 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec21074 ret <4 x i32> %res1075}1076 1077define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mem_mask0(ptr %vp, <4 x i32> %mask) {1078; CHECK-LABEL: test_masked_z_8xi32_to_4xi32_perm_mem_mask0:1079; CHECK: # %bb.0:1080; CHECK-NEXT: vmovaps 16(%rdi), %xmm11081; CHECK-NEXT: vshufps $7, (%rdi), %xmm1, %xmm1 # xmm1 = xmm1[3,1],mem[0,0]1082; CHECK-NEXT: vptestnmd %xmm0, %xmm0, %k11083; CHECK-NEXT: vmovdqa32 %xmm1, %xmm0 {%k1} {z}1084; CHECK-NEXT: retq1085 %vec = load <8 x i32>, ptr %vp1086 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 7, i32 5, i32 0, i32 0>1087 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1088 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer1089 ret <4 x i32> %res1090}1091 1092define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mem_mask1(ptr %vp, <4 x i32> %vec2, <4 x i32> %mask) {1093; CHECK-LABEL: test_masked_8xi32_to_4xi32_perm_mem_mask1:1094; CHECK: # %bb.0:1095; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm01096; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [5,0,0,3]1097; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k11098; CHECK-NEXT: vpermd (%rdi), %ymm2, %ymm0 {%k1}1099; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm01100; CHECK-NEXT: vzeroupper1101; CHECK-NEXT: retq1102 %vec = load <8 x i32>, ptr %vp1103 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 0, i32 0, i32 3>1104 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1105 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec21106 ret <4 x i32> %res1107}1108 1109define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mem_mask1(ptr %vp, <4 x i32> %mask) {1110; CHECK-LABEL: test_masked_z_8xi32_to_4xi32_perm_mem_mask1:1111; CHECK: # %bb.0:1112; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [5,0,0,3]1113; CHECK-NEXT: vptestnmd %xmm0, %xmm0, %k11114; CHECK-NEXT: vpermd (%rdi), %ymm1, %ymm0 {%k1} {z}1115; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm01116; CHECK-NEXT: vzeroupper1117; CHECK-NEXT: retq1118 %vec = load <8 x i32>, ptr %vp1119 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 0, i32 0, i32 3>1120 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1121 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer1122 ret <4 x i32> %res1123}1124 1125define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mem_mask2(ptr %vp, <4 x i32> %vec2, <4 x i32> %mask) {1126; CHECK-LABEL: test_masked_8xi32_to_4xi32_perm_mem_mask2:1127; CHECK: # %bb.0:1128; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm01129; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [4,3,3,4]1130; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k11131; CHECK-NEXT: vpermd (%rdi), %ymm2, %ymm0 {%k1}1132; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm01133; CHECK-NEXT: vzeroupper1134; CHECK-NEXT: retq1135 %vec = load <8 x i32>, ptr %vp1136 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 4, i32 3, i32 3, i32 4>1137 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1138 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec21139 ret <4 x i32> %res1140}1141 1142define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mem_mask2(ptr %vp, <4 x i32> %mask) {1143; CHECK-LABEL: test_masked_z_8xi32_to_4xi32_perm_mem_mask2:1144; CHECK: # %bb.0:1145; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [4,3,3,4]1146; CHECK-NEXT: vptestnmd %xmm0, %xmm0, %k11147; CHECK-NEXT: vpermd (%rdi), %ymm1, %ymm0 {%k1} {z}1148; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm01149; CHECK-NEXT: vzeroupper1150; CHECK-NEXT: retq1151 %vec = load <8 x i32>, ptr %vp1152 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 4, i32 3, i32 3, i32 4>1153 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1154 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer1155 ret <4 x i32> %res1156}1157 1158define <4 x i32> @test_8xi32_to_4xi32_perm_mem_mask3(ptr %vp) {1159; CHECK-LABEL: test_8xi32_to_4xi32_perm_mem_mask3:1160; CHECK: # %bb.0:1161; CHECK-NEXT: vpbroadcastq 8(%rdi), %xmm11162; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm0 = [5,1,2,7]1163; CHECK-NEXT: vpermi2d 16(%rdi), %xmm1, %xmm01164; CHECK-NEXT: retq1165 %vec = load <8 x i32>, ptr %vp1166 %res = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 3, i32 2, i32 7>1167 ret <4 x i32> %res1168}1169define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mem_mask3(ptr %vp, <4 x i32> %vec2, <4 x i32> %mask) {1170; CHECK-LABEL: test_masked_8xi32_to_4xi32_perm_mem_mask3:1171; CHECK: # %bb.0:1172; CHECK-NEXT: vpbroadcastq 8(%rdi), %xmm21173; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm3 = [5,1,2,7]1174; CHECK-NEXT: vpermi2d 16(%rdi), %xmm2, %xmm31175; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k11176; CHECK-NEXT: vmovdqa32 %xmm3, %xmm0 {%k1}1177; CHECK-NEXT: retq1178 %vec = load <8 x i32>, ptr %vp1179 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 3, i32 2, i32 7>1180 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1181 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec21182 ret <4 x i32> %res1183}1184 1185define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mem_mask3(ptr %vp, <4 x i32> %mask) {1186; CHECK-LABEL: test_masked_z_8xi32_to_4xi32_perm_mem_mask3:1187; CHECK: # %bb.0:1188; CHECK-NEXT: vpbroadcastq 8(%rdi), %xmm21189; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [5,1,2,7]1190; CHECK-NEXT: vptestnmd %xmm0, %xmm0, %k11191; CHECK-NEXT: vpermi2d 16(%rdi), %xmm2, %xmm1 {%k1} {z}1192; CHECK-NEXT: vmovdqa %xmm1, %xmm01193; CHECK-NEXT: retq1194 %vec = load <8 x i32>, ptr %vp1195 %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 3, i32 2, i32 7>1196 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1197 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer1198 ret <4 x i32> %res1199}1200 1201define <8 x i32> @test_16xi32_to_8xi32_perm_mask0(<16 x i32> %vec) {1202; CHECK-LABEL: test_16xi32_to_8xi32_perm_mask0:1203; CHECK: # %bb.0:1204; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,13,11,14,7,10,1,6]1205; CHECK-NEXT: vpermps %zmm0, %zmm1, %zmm01206; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01207; CHECK-NEXT: retq1208 %res = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 1, i32 13, i32 11, i32 14, i32 7, i32 10, i32 1, i32 6>1209 ret <8 x i32> %res1210}1211define <8 x i32> @test_masked_16xi32_to_8xi32_perm_mask0(<16 x i32> %vec, <8 x i32> %vec2, <8 x i32> %mask) {1212; CHECK-LABEL: test_masked_16xi32_to_8xi32_perm_mask0:1213; CHECK: # %bb.0:1214; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm11215; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm3 = [1,13,11,14,7,10,1,6]1216; CHECK-NEXT: vptestnmd %ymm2, %ymm2, %k11217; CHECK-NEXT: vpermd %zmm0, %zmm3, %zmm1 {%k1}1218; CHECK-NEXT: vmovdqa %ymm1, %ymm01219; CHECK-NEXT: retq1220 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 1, i32 13, i32 11, i32 14, i32 7, i32 10, i32 1, i32 6>1221 %cmp = icmp eq <8 x i32> %mask, zeroinitializer1222 %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec21223 ret <8 x i32> %res1224}1225 1226define <8 x i32> @test_masked_z_16xi32_to_8xi32_perm_mask0(<16 x i32> %vec, <8 x i32> %mask) {1227; CHECK-LABEL: test_masked_z_16xi32_to_8xi32_perm_mask0:1228; CHECK: # %bb.0:1229; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [1,13,11,14,7,10,1,6]1230; CHECK-NEXT: vptestnmd %ymm1, %ymm1, %k11231; CHECK-NEXT: vpermd %zmm0, %zmm2, %zmm0 {%k1} {z}1232; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01233; CHECK-NEXT: retq1234 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 1, i32 13, i32 11, i32 14, i32 7, i32 10, i32 1, i32 6>1235 %cmp = icmp eq <8 x i32> %mask, zeroinitializer1236 %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer1237 ret <8 x i32> %res1238}1239define <8 x i32> @test_masked_16xi32_to_8xi32_perm_mask1(<16 x i32> %vec, <8 x i32> %vec2, <8 x i32> %mask) {1240; CHECK-LABEL: test_masked_16xi32_to_8xi32_perm_mask1:1241; CHECK: # %bb.0:1242; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm11243; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm3 = [3,0,15,3,2,3,6,8]1244; CHECK-NEXT: vptestnmd %ymm2, %ymm2, %k11245; CHECK-NEXT: vpermd %zmm0, %zmm3, %zmm1 {%k1}1246; CHECK-NEXT: vmovdqa %ymm1, %ymm01247; CHECK-NEXT: retq1248 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 3, i32 0, i32 15, i32 3, i32 2, i32 3, i32 6, i32 8>1249 %cmp = icmp eq <8 x i32> %mask, zeroinitializer1250 %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec21251 ret <8 x i32> %res1252}1253 1254define <8 x i32> @test_masked_z_16xi32_to_8xi32_perm_mask1(<16 x i32> %vec, <8 x i32> %mask) {1255; CHECK-LABEL: test_masked_z_16xi32_to_8xi32_perm_mask1:1256; CHECK: # %bb.0:1257; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [3,0,15,3,2,3,6,8]1258; CHECK-NEXT: vptestnmd %ymm1, %ymm1, %k11259; CHECK-NEXT: vpermd %zmm0, %zmm2, %zmm0 {%k1} {z}1260; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01261; CHECK-NEXT: retq1262 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 3, i32 0, i32 15, i32 3, i32 2, i32 3, i32 6, i32 8>1263 %cmp = icmp eq <8 x i32> %mask, zeroinitializer1264 %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer1265 ret <8 x i32> %res1266}1267define <8 x i32> @test_masked_16xi32_to_8xi32_perm_mask2(<16 x i32> %vec, <8 x i32> %vec2, <8 x i32> %mask) {1268; CHECK-LABEL: test_masked_16xi32_to_8xi32_perm_mask2:1269; CHECK: # %bb.0:1270; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm11271; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm3 = [2,15,15,2,6,10,14,7]1272; CHECK-NEXT: vptestnmd %ymm2, %ymm2, %k11273; CHECK-NEXT: vpermd %zmm0, %zmm3, %zmm1 {%k1}1274; CHECK-NEXT: vmovdqa %ymm1, %ymm01275; CHECK-NEXT: retq1276 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 2, i32 15, i32 15, i32 2, i32 6, i32 10, i32 14, i32 7>1277 %cmp = icmp eq <8 x i32> %mask, zeroinitializer1278 %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec21279 ret <8 x i32> %res1280}1281 1282define <8 x i32> @test_masked_z_16xi32_to_8xi32_perm_mask2(<16 x i32> %vec, <8 x i32> %mask) {1283; CHECK-LABEL: test_masked_z_16xi32_to_8xi32_perm_mask2:1284; CHECK: # %bb.0:1285; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [2,15,15,2,6,10,14,7]1286; CHECK-NEXT: vptestnmd %ymm1, %ymm1, %k11287; CHECK-NEXT: vpermd %zmm0, %zmm2, %zmm0 {%k1} {z}1288; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01289; CHECK-NEXT: retq1290 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 2, i32 15, i32 15, i32 2, i32 6, i32 10, i32 14, i32 7>1291 %cmp = icmp eq <8 x i32> %mask, zeroinitializer1292 %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer1293 ret <8 x i32> %res1294}1295define <8 x i32> @test_16xi32_to_8xi32_perm_mask3(<16 x i32> %vec) {1296; CHECK-LABEL: test_16xi32_to_8xi32_perm_mask3:1297; CHECK: # %bb.0:1298; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [14,5,7,7,10,3,9,3]1299; CHECK-NEXT: vpermps %zmm0, %zmm1, %zmm01300; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01301; CHECK-NEXT: retq1302 %res = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 14, i32 5, i32 7, i32 7, i32 10, i32 3, i32 9, i32 3>1303 ret <8 x i32> %res1304}1305define <8 x i32> @test_masked_16xi32_to_8xi32_perm_mask3(<16 x i32> %vec, <8 x i32> %vec2, <8 x i32> %mask) {1306; CHECK-LABEL: test_masked_16xi32_to_8xi32_perm_mask3:1307; CHECK: # %bb.0:1308; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm11309; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm3 = [14,5,7,7,10,3,9,3]1310; CHECK-NEXT: vptestnmd %ymm2, %ymm2, %k11311; CHECK-NEXT: vpermd %zmm0, %zmm3, %zmm1 {%k1}1312; CHECK-NEXT: vmovdqa %ymm1, %ymm01313; CHECK-NEXT: retq1314 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 14, i32 5, i32 7, i32 7, i32 10, i32 3, i32 9, i32 3>1315 %cmp = icmp eq <8 x i32> %mask, zeroinitializer1316 %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec21317 ret <8 x i32> %res1318}1319 1320define <8 x i32> @test_masked_z_16xi32_to_8xi32_perm_mask3(<16 x i32> %vec, <8 x i32> %mask) {1321; CHECK-LABEL: test_masked_z_16xi32_to_8xi32_perm_mask3:1322; CHECK: # %bb.0:1323; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [14,5,7,7,10,3,9,3]1324; CHECK-NEXT: vptestnmd %ymm1, %ymm1, %k11325; CHECK-NEXT: vpermd %zmm0, %zmm2, %zmm0 {%k1} {z}1326; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01327; CHECK-NEXT: retq1328 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 14, i32 5, i32 7, i32 7, i32 10, i32 3, i32 9, i32 3>1329 %cmp = icmp eq <8 x i32> %mask, zeroinitializer1330 %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer1331 ret <8 x i32> %res1332}1333define <4 x i32> @test_16xi32_to_4xi32_perm_mask0(<16 x i32> %vec) {1334; CHECK-LABEL: test_16xi32_to_4xi32_perm_mask0:1335; CHECK: # %bb.0:1336; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,2,4,12,4,6,4,12]1337; CHECK-NEXT: vpermps %zmm0, %zmm1, %zmm01338; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01339; CHECK-NEXT: vzeroupper1340; CHECK-NEXT: retq1341 %res = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 12>1342 ret <4 x i32> %res1343}1344define <4 x i32> @test_masked_16xi32_to_4xi32_perm_mask0(<16 x i32> %vec, <4 x i32> %vec2, <4 x i32> %mask) {1345; CHECK-LABEL: test_masked_16xi32_to_4xi32_perm_mask0:1346; CHECK: # %bb.0:1347; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm11348; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,2,4,12,4,6,4,12]1349; CHECK-NEXT: vptestnmd %xmm2, %xmm2, %k11350; CHECK-NEXT: vpermd %zmm0, %zmm3, %zmm1 {%k1}1351; CHECK-NEXT: vmovdqa %xmm1, %xmm01352; CHECK-NEXT: vzeroupper1353; CHECK-NEXT: retq1354 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 12>1355 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1356 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec21357 ret <4 x i32> %res1358}1359 1360define <4 x i32> @test_masked_z_16xi32_to_4xi32_perm_mask0(<16 x i32> %vec, <4 x i32> %mask) {1361; CHECK-LABEL: test_masked_z_16xi32_to_4xi32_perm_mask0:1362; CHECK: # %bb.0:1363; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,2,4,12,4,6,4,12]1364; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k11365; CHECK-NEXT: vpermd %zmm0, %zmm2, %zmm0 {%k1} {z}1366; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01367; CHECK-NEXT: vzeroupper1368; CHECK-NEXT: retq1369 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 12>1370 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1371 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer1372 ret <4 x i32> %res1373}1374define <4 x i32> @test_masked_16xi32_to_4xi32_perm_mask1(<16 x i32> %vec, <4 x i32> %vec2, <4 x i32> %mask) {1375; CHECK-LABEL: test_masked_16xi32_to_4xi32_perm_mask1:1376; CHECK: # %bb.0:1377; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm11378; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm3 = [5,1,3,4]1379; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm01380; CHECK-NEXT: vptestnmd %xmm2, %xmm2, %k11381; CHECK-NEXT: vpermd %ymm0, %ymm3, %ymm1 {%k1}1382; CHECK-NEXT: vmovdqa %xmm1, %xmm01383; CHECK-NEXT: vzeroupper1384; CHECK-NEXT: retq1385 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 13, i32 9, i32 11, i32 12>1386 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1387 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec21388 ret <4 x i32> %res1389}1390 1391define <4 x i32> @test_masked_z_16xi32_to_4xi32_perm_mask1(<16 x i32> %vec, <4 x i32> %mask) {1392; CHECK-LABEL: test_masked_z_16xi32_to_4xi32_perm_mask1:1393; CHECK: # %bb.0:1394; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [5,1,3,4]1395; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm01396; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k11397; CHECK-NEXT: vpermd %ymm0, %ymm2, %ymm0 {%k1} {z}1398; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm01399; CHECK-NEXT: vzeroupper1400; CHECK-NEXT: retq1401 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 13, i32 9, i32 11, i32 12>1402 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1403 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer1404 ret <4 x i32> %res1405}1406define <4 x i32> @test_masked_16xi32_to_4xi32_perm_mask2(<16 x i32> %vec, <4 x i32> %vec2, <4 x i32> %mask) {1407; CHECK-LABEL: test_masked_16xi32_to_4xi32_perm_mask2:1408; CHECK: # %bb.0:1409; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm11410; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm3 = [1,1,13,0]1411; CHECK-NEXT: vptestnmd %xmm2, %xmm2, %k11412; CHECK-NEXT: vpermd %zmm0, %zmm3, %zmm1 {%k1}1413; CHECK-NEXT: vmovdqa %xmm1, %xmm01414; CHECK-NEXT: vzeroupper1415; CHECK-NEXT: retq1416 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 1, i32 1, i32 13, i32 0>1417 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1418 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec21419 ret <4 x i32> %res1420}1421 1422define <4 x i32> @test_masked_z_16xi32_to_4xi32_perm_mask2(<16 x i32> %vec, <4 x i32> %mask) {1423; CHECK-LABEL: test_masked_z_16xi32_to_4xi32_perm_mask2:1424; CHECK: # %bb.0:1425; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [1,1,13,0]1426; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k11427; CHECK-NEXT: vpermd %zmm0, %zmm2, %zmm0 {%k1} {z}1428; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01429; CHECK-NEXT: vzeroupper1430; CHECK-NEXT: retq1431 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 1, i32 1, i32 13, i32 0>1432 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1433 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer1434 ret <4 x i32> %res1435}1436define <4 x i32> @test_16xi32_to_4xi32_perm_mask3(<16 x i32> %vec) {1437; CHECK-LABEL: test_16xi32_to_4xi32_perm_mask3:1438; CHECK: # %bb.0:1439; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [3,0,0,13]1440; CHECK-NEXT: vpermps %zmm0, %zmm1, %zmm01441; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01442; CHECK-NEXT: vzeroupper1443; CHECK-NEXT: retq1444 %res = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 3, i32 0, i32 0, i32 13>1445 ret <4 x i32> %res1446}1447define <4 x i32> @test_masked_16xi32_to_4xi32_perm_mask3(<16 x i32> %vec, <4 x i32> %vec2, <4 x i32> %mask) {1448; CHECK-LABEL: test_masked_16xi32_to_4xi32_perm_mask3:1449; CHECK: # %bb.0:1450; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm11451; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm3 = [3,0,0,13]1452; CHECK-NEXT: vptestnmd %xmm2, %xmm2, %k11453; CHECK-NEXT: vpermd %zmm0, %zmm3, %zmm1 {%k1}1454; CHECK-NEXT: vmovdqa %xmm1, %xmm01455; CHECK-NEXT: vzeroupper1456; CHECK-NEXT: retq1457 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 3, i32 0, i32 0, i32 13>1458 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1459 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec21460 ret <4 x i32> %res1461}1462 1463define <4 x i32> @test_masked_z_16xi32_to_4xi32_perm_mask3(<16 x i32> %vec, <4 x i32> %mask) {1464; CHECK-LABEL: test_masked_z_16xi32_to_4xi32_perm_mask3:1465; CHECK: # %bb.0:1466; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [3,0,0,13]1467; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k11468; CHECK-NEXT: vpermd %zmm0, %zmm2, %zmm0 {%k1} {z}1469; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01470; CHECK-NEXT: vzeroupper1471; CHECK-NEXT: retq1472 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 3, i32 0, i32 0, i32 13>1473 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1474 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer1475 ret <4 x i32> %res1476}1477define <8 x i32> @test_16xi32_to_8xi32_perm_mem_mask0(ptr %vp) {1478; CHECK-LABEL: test_16xi32_to_8xi32_perm_mem_mask0:1479; CHECK: # %bb.0:1480; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm0 = [7,0,6,0,1,2,4,4]1481; CHECK-NEXT: vpermps 32(%rdi), %ymm0, %ymm01482; CHECK-NEXT: retq1483 %vec = load <16 x i32>, ptr %vp1484 %res = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 15, i32 8, i32 14, i32 8, i32 9, i32 10, i32 12, i32 12>1485 ret <8 x i32> %res1486}1487define <8 x i32> @test_masked_16xi32_to_8xi32_perm_mem_mask0(ptr %vp, <8 x i32> %vec2, <8 x i32> %mask) {1488; CHECK-LABEL: test_masked_16xi32_to_8xi32_perm_mem_mask0:1489; CHECK: # %bb.0:1490; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [7,0,6,0,1,2,4,4]1491; CHECK-NEXT: vptestnmd %ymm1, %ymm1, %k11492; CHECK-NEXT: vpermd 32(%rdi), %ymm2, %ymm0 {%k1}1493; CHECK-NEXT: retq1494 %vec = load <16 x i32>, ptr %vp1495 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 15, i32 8, i32 14, i32 8, i32 9, i32 10, i32 12, i32 12>1496 %cmp = icmp eq <8 x i32> %mask, zeroinitializer1497 %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec21498 ret <8 x i32> %res1499}1500 1501define <8 x i32> @test_masked_z_16xi32_to_8xi32_perm_mem_mask0(ptr %vp, <8 x i32> %mask) {1502; CHECK-LABEL: test_masked_z_16xi32_to_8xi32_perm_mem_mask0:1503; CHECK: # %bb.0:1504; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [7,0,6,0,1,2,4,4]1505; CHECK-NEXT: vptestnmd %ymm0, %ymm0, %k11506; CHECK-NEXT: vpermd 32(%rdi), %ymm1, %ymm0 {%k1} {z}1507; CHECK-NEXT: retq1508 %vec = load <16 x i32>, ptr %vp1509 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 15, i32 8, i32 14, i32 8, i32 9, i32 10, i32 12, i32 12>1510 %cmp = icmp eq <8 x i32> %mask, zeroinitializer1511 %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer1512 ret <8 x i32> %res1513}1514 1515define <8 x i32> @test_masked_16xi32_to_8xi32_perm_mem_mask1(ptr %vp, <8 x i32> %vec2, <8 x i32> %mask) {1516; CHECK-LABEL: test_masked_16xi32_to_8xi32_perm_mem_mask1:1517; CHECK: # %bb.0:1518; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01519; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [15,11,14,3,8,9,13,7]1520; CHECK-NEXT: vptestnmd %ymm1, %ymm1, %k11521; CHECK-NEXT: vpermd (%rdi), %zmm2, %zmm0 {%k1}1522; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01523; CHECK-NEXT: retq1524 %vec = load <16 x i32>, ptr %vp1525 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 15, i32 11, i32 14, i32 3, i32 8, i32 9, i32 13, i32 7>1526 %cmp = icmp eq <8 x i32> %mask, zeroinitializer1527 %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec21528 ret <8 x i32> %res1529}1530 1531define <8 x i32> @test_masked_z_16xi32_to_8xi32_perm_mem_mask1(ptr %vp, <8 x i32> %mask) {1532; CHECK-LABEL: test_masked_z_16xi32_to_8xi32_perm_mem_mask1:1533; CHECK: # %bb.0:1534; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [15,11,14,3,8,9,13,7]1535; CHECK-NEXT: vptestnmd %ymm0, %ymm0, %k11536; CHECK-NEXT: vpermd (%rdi), %zmm1, %zmm0 {%k1} {z}1537; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01538; CHECK-NEXT: retq1539 %vec = load <16 x i32>, ptr %vp1540 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 15, i32 11, i32 14, i32 3, i32 8, i32 9, i32 13, i32 7>1541 %cmp = icmp eq <8 x i32> %mask, zeroinitializer1542 %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer1543 ret <8 x i32> %res1544}1545 1546define <8 x i32> @test_masked_16xi32_to_8xi32_perm_mem_mask2(ptr %vp, <8 x i32> %vec2, <8 x i32> %mask) {1547; CHECK-LABEL: test_masked_16xi32_to_8xi32_perm_mem_mask2:1548; CHECK: # %bb.0:1549; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01550; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [12,6,9,13,12,10,0,2]1551; CHECK-NEXT: vptestnmd %ymm1, %ymm1, %k11552; CHECK-NEXT: vpermd (%rdi), %zmm2, %zmm0 {%k1}1553; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01554; CHECK-NEXT: retq1555 %vec = load <16 x i32>, ptr %vp1556 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 12, i32 6, i32 9, i32 13, i32 12, i32 10, i32 0, i32 2>1557 %cmp = icmp eq <8 x i32> %mask, zeroinitializer1558 %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec21559 ret <8 x i32> %res1560}1561 1562define <8 x i32> @test_masked_z_16xi32_to_8xi32_perm_mem_mask2(ptr %vp, <8 x i32> %mask) {1563; CHECK-LABEL: test_masked_z_16xi32_to_8xi32_perm_mem_mask2:1564; CHECK: # %bb.0:1565; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [12,6,9,13,12,10,0,2]1566; CHECK-NEXT: vptestnmd %ymm0, %ymm0, %k11567; CHECK-NEXT: vpermd (%rdi), %zmm1, %zmm0 {%k1} {z}1568; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01569; CHECK-NEXT: retq1570 %vec = load <16 x i32>, ptr %vp1571 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 12, i32 6, i32 9, i32 13, i32 12, i32 10, i32 0, i32 2>1572 %cmp = icmp eq <8 x i32> %mask, zeroinitializer1573 %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer1574 ret <8 x i32> %res1575}1576 1577define <8 x i32> @test_16xi32_to_8xi32_perm_mem_mask3(ptr %vp) {1578; CHECK-LABEL: test_16xi32_to_8xi32_perm_mem_mask3:1579; CHECK: # %bb.0:1580; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm0 = [8,4,1,13,15,4,6,12]1581; CHECK-NEXT: vpermps (%rdi), %zmm0, %zmm01582; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01583; CHECK-NEXT: retq1584 %vec = load <16 x i32>, ptr %vp1585 %res = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 8, i32 4, i32 1, i32 13, i32 15, i32 4, i32 6, i32 12>1586 ret <8 x i32> %res1587}1588define <8 x i32> @test_masked_16xi32_to_8xi32_perm_mem_mask3(ptr %vp, <8 x i32> %vec2, <8 x i32> %mask) {1589; CHECK-LABEL: test_masked_16xi32_to_8xi32_perm_mem_mask3:1590; CHECK: # %bb.0:1591; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01592; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [8,4,1,13,15,4,6,12]1593; CHECK-NEXT: vptestnmd %ymm1, %ymm1, %k11594; CHECK-NEXT: vpermd (%rdi), %zmm2, %zmm0 {%k1}1595; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01596; CHECK-NEXT: retq1597 %vec = load <16 x i32>, ptr %vp1598 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 8, i32 4, i32 1, i32 13, i32 15, i32 4, i32 6, i32 12>1599 %cmp = icmp eq <8 x i32> %mask, zeroinitializer1600 %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec21601 ret <8 x i32> %res1602}1603 1604define <8 x i32> @test_masked_z_16xi32_to_8xi32_perm_mem_mask3(ptr %vp, <8 x i32> %mask) {1605; CHECK-LABEL: test_masked_z_16xi32_to_8xi32_perm_mem_mask3:1606; CHECK: # %bb.0:1607; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [8,4,1,13,15,4,6,12]1608; CHECK-NEXT: vptestnmd %ymm0, %ymm0, %k11609; CHECK-NEXT: vpermd (%rdi), %zmm1, %zmm0 {%k1} {z}1610; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01611; CHECK-NEXT: retq1612 %vec = load <16 x i32>, ptr %vp1613 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <8 x i32> <i32 8, i32 4, i32 1, i32 13, i32 15, i32 4, i32 6, i32 12>1614 %cmp = icmp eq <8 x i32> %mask, zeroinitializer1615 %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer1616 ret <8 x i32> %res1617}1618 1619define <4 x i32> @test_16xi32_to_4xi32_perm_mem_mask0(ptr %vp) {1620; CHECK-LABEL: test_16xi32_to_4xi32_perm_mem_mask0:1621; CHECK: # %bb.0:1622; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm0 = [13,0,0,6]1623; CHECK-NEXT: vpermps (%rdi), %zmm0, %zmm01624; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01625; CHECK-NEXT: vzeroupper1626; CHECK-NEXT: retq1627 %vec = load <16 x i32>, ptr %vp1628 %res = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 13, i32 0, i32 0, i32 6>1629 ret <4 x i32> %res1630}1631define <4 x i32> @test_masked_16xi32_to_4xi32_perm_mem_mask0(ptr %vp, <4 x i32> %vec2, <4 x i32> %mask) {1632; CHECK-LABEL: test_masked_16xi32_to_4xi32_perm_mem_mask0:1633; CHECK: # %bb.0:1634; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm01635; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [13,0,0,6]1636; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k11637; CHECK-NEXT: vpermd (%rdi), %zmm2, %zmm0 {%k1}1638; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01639; CHECK-NEXT: vzeroupper1640; CHECK-NEXT: retq1641 %vec = load <16 x i32>, ptr %vp1642 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 13, i32 0, i32 0, i32 6>1643 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1644 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec21645 ret <4 x i32> %res1646}1647 1648define <4 x i32> @test_masked_z_16xi32_to_4xi32_perm_mem_mask0(ptr %vp, <4 x i32> %mask) {1649; CHECK-LABEL: test_masked_z_16xi32_to_4xi32_perm_mem_mask0:1650; CHECK: # %bb.0:1651; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [13,0,0,6]1652; CHECK-NEXT: vptestnmd %xmm0, %xmm0, %k11653; CHECK-NEXT: vpermd (%rdi), %zmm1, %zmm0 {%k1} {z}1654; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01655; CHECK-NEXT: vzeroupper1656; CHECK-NEXT: retq1657 %vec = load <16 x i32>, ptr %vp1658 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 13, i32 0, i32 0, i32 6>1659 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1660 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer1661 ret <4 x i32> %res1662}1663 1664define <4 x i32> @test_masked_16xi32_to_4xi32_perm_mem_mask1(ptr %vp, <4 x i32> %vec2, <4 x i32> %mask) {1665; CHECK-LABEL: test_masked_16xi32_to_4xi32_perm_mem_mask1:1666; CHECK: # %bb.0:1667; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm01668; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [7,13,11,10,7,13,15,14]1669; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k11670; CHECK-NEXT: vpermd (%rdi), %zmm2, %zmm0 {%k1}1671; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01672; CHECK-NEXT: vzeroupper1673; CHECK-NEXT: retq1674 %vec = load <16 x i32>, ptr %vp1675 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 7, i32 13, i32 11, i32 10>1676 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1677 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec21678 ret <4 x i32> %res1679}1680 1681define <4 x i32> @test_masked_z_16xi32_to_4xi32_perm_mem_mask1(ptr %vp, <4 x i32> %mask) {1682; CHECK-LABEL: test_masked_z_16xi32_to_4xi32_perm_mem_mask1:1683; CHECK: # %bb.0:1684; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [7,13,11,10,7,13,15,14]1685; CHECK-NEXT: vptestnmd %xmm0, %xmm0, %k11686; CHECK-NEXT: vpermd (%rdi), %zmm1, %zmm0 {%k1} {z}1687; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01688; CHECK-NEXT: vzeroupper1689; CHECK-NEXT: retq1690 %vec = load <16 x i32>, ptr %vp1691 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 7, i32 13, i32 11, i32 10>1692 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1693 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer1694 ret <4 x i32> %res1695}1696 1697define <4 x i32> @test_masked_16xi32_to_4xi32_perm_mem_mask2(ptr %vp, <4 x i32> %vec2, <4 x i32> %mask) {1698; CHECK-LABEL: test_masked_16xi32_to_4xi32_perm_mem_mask2:1699; CHECK: # %bb.0:1700; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm01701; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [2,15,6,9]1702; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k11703; CHECK-NEXT: vpermd (%rdi), %zmm2, %zmm0 {%k1}1704; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01705; CHECK-NEXT: vzeroupper1706; CHECK-NEXT: retq1707 %vec = load <16 x i32>, ptr %vp1708 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 2, i32 15, i32 6, i32 9>1709 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1710 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec21711 ret <4 x i32> %res1712}1713 1714define <4 x i32> @test_masked_z_16xi32_to_4xi32_perm_mem_mask2(ptr %vp, <4 x i32> %mask) {1715; CHECK-LABEL: test_masked_z_16xi32_to_4xi32_perm_mem_mask2:1716; CHECK: # %bb.0:1717; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [2,15,6,9]1718; CHECK-NEXT: vptestnmd %xmm0, %xmm0, %k11719; CHECK-NEXT: vpermd (%rdi), %zmm1, %zmm0 {%k1} {z}1720; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01721; CHECK-NEXT: vzeroupper1722; CHECK-NEXT: retq1723 %vec = load <16 x i32>, ptr %vp1724 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 2, i32 15, i32 6, i32 9>1725 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1726 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer1727 ret <4 x i32> %res1728}1729 1730define <4 x i32> @test_16xi32_to_4xi32_perm_mem_mask3(ptr %vp) {1731; CHECK-LABEL: test_16xi32_to_4xi32_perm_mem_mask3:1732; CHECK: # %bb.0:1733; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm0 = [6,0,7,2]1734; CHECK-NEXT: vpermps (%rdi), %ymm0, %ymm01735; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm01736; CHECK-NEXT: vzeroupper1737; CHECK-NEXT: retq1738 %vec = load <16 x i32>, ptr %vp1739 %res = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 6, i32 0, i32 7, i32 2>1740 ret <4 x i32> %res1741}1742define <4 x i32> @test_masked_16xi32_to_4xi32_perm_mem_mask3(ptr %vp, <4 x i32> %vec2, <4 x i32> %mask) {1743; CHECK-LABEL: test_masked_16xi32_to_4xi32_perm_mem_mask3:1744; CHECK: # %bb.0:1745; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm01746; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [6,0,7,2]1747; CHECK-NEXT: vptestnmd %xmm1, %xmm1, %k11748; CHECK-NEXT: vpermd (%rdi), %ymm2, %ymm0 {%k1}1749; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm01750; CHECK-NEXT: vzeroupper1751; CHECK-NEXT: retq1752 %vec = load <16 x i32>, ptr %vp1753 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 6, i32 0, i32 7, i32 2>1754 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1755 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec21756 ret <4 x i32> %res1757}1758 1759define <4 x i32> @test_masked_z_16xi32_to_4xi32_perm_mem_mask3(ptr %vp, <4 x i32> %mask) {1760; CHECK-LABEL: test_masked_z_16xi32_to_4xi32_perm_mem_mask3:1761; CHECK: # %bb.0:1762; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [6,0,7,2]1763; CHECK-NEXT: vptestnmd %xmm0, %xmm0, %k11764; CHECK-NEXT: vpermd (%rdi), %ymm1, %ymm0 {%k1} {z}1765; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm01766; CHECK-NEXT: vzeroupper1767; CHECK-NEXT: retq1768 %vec = load <16 x i32>, ptr %vp1769 %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 6, i32 0, i32 7, i32 2>1770 %cmp = icmp eq <4 x i32> %mask, zeroinitializer1771 %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer1772 ret <4 x i32> %res1773}1774 1775define <4 x i32> @test_16xi32_to_4xi32_perm_mask9(<16 x i32> %vec) {1776; CHECK-FAST-LABEL: test_16xi32_to_4xi32_perm_mask9:1777; CHECK-FAST: # %bb.0:1778; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} xmm1 = [12,9,4,10]1779; CHECK-FAST-NEXT: vpermps %zmm0, %zmm1, %zmm01780; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm01781; CHECK-FAST-NEXT: vzeroupper1782; CHECK-FAST-NEXT: retq1783;1784; CHECK-FAST-PERLANE-LABEL: test_16xi32_to_4xi32_perm_mask9:1785; CHECK-FAST-PERLANE: # %bb.0:1786; CHECK-FAST-PERLANE-NEXT: vpmovsxbd {{.*#+}} xmm1 = [4,1,0,2]1787; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm21788; CHECK-FAST-PERLANE-NEXT: vpermd %ymm2, %ymm1, %ymm11789; CHECK-FAST-PERLANE-NEXT: vextracti128 $1, %ymm0, %xmm21790; CHECK-FAST-PERLANE-NEXT: vpmovsxbd {{.*#+}} xmm0 = [0,1,4,3]1791; CHECK-FAST-PERLANE-NEXT: vpermi2d %xmm2, %xmm1, %xmm01792; CHECK-FAST-PERLANE-NEXT: vzeroupper1793; CHECK-FAST-PERLANE-NEXT: retq1794 %res = shufflevector <16 x i32> %vec, <16 x i32> undef, <4 x i32> <i32 12, i32 9, i32 4, i32 10>1795 ret <4 x i32> %res1796}1797 1798define <2 x i64> @test_4xi64_to_2xi64_perm_mask0(<4 x i64> %vec) {1799; CHECK-LABEL: test_4xi64_to_2xi64_perm_mask0:1800; CHECK: # %bb.0:1801; CHECK-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,0,2,3]1802; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm01803; CHECK-NEXT: vzeroupper1804; CHECK-NEXT: retq1805 %res = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 2, i32 0>1806 ret <2 x i64> %res1807}1808define <2 x i64> @test_masked_4xi64_to_2xi64_perm_mask0(<4 x i64> %vec, <2 x i64> %vec2, <2 x i64> %mask) {1809; CHECK-LABEL: test_masked_4xi64_to_2xi64_perm_mask0:1810; CHECK: # %bb.0:1811; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm11812; CHECK-NEXT: vptestnmq %xmm2, %xmm2, %k11813; CHECK-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,0,2,3]1814; CHECK-NEXT: vmovdqa %xmm1, %xmm01815; CHECK-NEXT: vzeroupper1816; CHECK-NEXT: retq1817 %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 2, i32 0>1818 %cmp = icmp eq <2 x i64> %mask, zeroinitializer1819 %res = select <2 x i1> %cmp, <2 x i64> %shuf, <2 x i64> %vec21820 ret <2 x i64> %res1821}1822 1823define <2 x i64> @test_masked_z_4xi64_to_2xi64_perm_mask0(<4 x i64> %vec, <2 x i64> %mask) {1824; CHECK-LABEL: test_masked_z_4xi64_to_2xi64_perm_mask0:1825; CHECK: # %bb.0:1826; CHECK-NEXT: vptestnmq %xmm1, %xmm1, %k11827; CHECK-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,0,2,3]1828; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm01829; CHECK-NEXT: vzeroupper1830; CHECK-NEXT: retq1831 %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 2, i32 0>1832 %cmp = icmp eq <2 x i64> %mask, zeroinitializer1833 %res = select <2 x i1> %cmp, <2 x i64> %shuf, <2 x i64> zeroinitializer1834 ret <2 x i64> %res1835}1836define <2 x i64> @test_masked_4xi64_to_2xi64_perm_mask1(<4 x i64> %vec, <2 x i64> %vec2, <2 x i64> %mask) {1837; CHECK-LABEL: test_masked_4xi64_to_2xi64_perm_mask1:1838; CHECK: # %bb.0:1839; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm11840; CHECK-NEXT: vptestnmq %xmm2, %xmm2, %k11841; CHECK-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,1,2,3]1842; CHECK-NEXT: vmovdqa %xmm1, %xmm01843; CHECK-NEXT: vzeroupper1844; CHECK-NEXT: retq1845 %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 2, i32 1>1846 %cmp = icmp eq <2 x i64> %mask, zeroinitializer1847 %res = select <2 x i1> %cmp, <2 x i64> %shuf, <2 x i64> %vec21848 ret <2 x i64> %res1849}1850 1851define <2 x i64> @test_masked_z_4xi64_to_2xi64_perm_mask1(<4 x i64> %vec, <2 x i64> %mask) {1852; CHECK-LABEL: test_masked_z_4xi64_to_2xi64_perm_mask1:1853; CHECK: # %bb.0:1854; CHECK-NEXT: vptestnmq %xmm1, %xmm1, %k11855; CHECK-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,1,2,3]1856; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm01857; CHECK-NEXT: vzeroupper1858; CHECK-NEXT: retq1859 %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 2, i32 1>1860 %cmp = icmp eq <2 x i64> %mask, zeroinitializer1861 %res = select <2 x i1> %cmp, <2 x i64> %shuf, <2 x i64> zeroinitializer1862 ret <2 x i64> %res1863}1864define <2 x i64> @test_4xi64_to_2xi64_perm_mem_mask0(ptr %vp) {1865; CHECK-LABEL: test_4xi64_to_2xi64_perm_mem_mask0:1866; CHECK: # %bb.0:1867; CHECK-NEXT: vmovaps (%rdi), %xmm01868; CHECK-NEXT: vunpckhpd 16(%rdi), %xmm0, %xmm0 # xmm0 = xmm0[1],mem[1]1869; CHECK-NEXT: retq1870 %vec = load <4 x i64>, ptr %vp1871 %res = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 1, i32 3>1872 ret <2 x i64> %res1873}1874define <2 x i64> @test_masked_4xi64_to_2xi64_perm_mem_mask0(ptr %vp, <2 x i64> %vec2, <2 x i64> %mask) {1875; CHECK-LABEL: test_masked_4xi64_to_2xi64_perm_mem_mask0:1876; CHECK: # %bb.0:1877; CHECK-NEXT: vmovdqa (%rdi), %xmm21878; CHECK-NEXT: vptestnmq %xmm1, %xmm1, %k11879; CHECK-NEXT: vpunpckhqdq 16(%rdi), %xmm2, %xmm0 {%k1} # xmm0 {%k1} = xmm2[1],mem[1]1880; CHECK-NEXT: retq1881 %vec = load <4 x i64>, ptr %vp1882 %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 1, i32 3>1883 %cmp = icmp eq <2 x i64> %mask, zeroinitializer1884 %res = select <2 x i1> %cmp, <2 x i64> %shuf, <2 x i64> %vec21885 ret <2 x i64> %res1886}1887 1888define <2 x i64> @test_masked_z_4xi64_to_2xi64_perm_mem_mask0(ptr %vp, <2 x i64> %mask) {1889; CHECK-LABEL: test_masked_z_4xi64_to_2xi64_perm_mem_mask0:1890; CHECK: # %bb.0:1891; CHECK-NEXT: vmovdqa (%rdi), %xmm11892; CHECK-NEXT: vptestnmq %xmm0, %xmm0, %k11893; CHECK-NEXT: vpunpckhqdq 16(%rdi), %xmm1, %xmm0 {%k1} {z} # xmm0 {%k1} {z} = xmm1[1],mem[1]1894; CHECK-NEXT: retq1895 %vec = load <4 x i64>, ptr %vp1896 %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 1, i32 3>1897 %cmp = icmp eq <2 x i64> %mask, zeroinitializer1898 %res = select <2 x i1> %cmp, <2 x i64> %shuf, <2 x i64> zeroinitializer1899 ret <2 x i64> %res1900}1901 1902define <2 x i64> @test_masked_4xi64_to_2xi64_perm_mem_mask1(ptr %vp, <2 x i64> %vec2, <2 x i64> %mask) {1903; CHECK-LABEL: test_masked_4xi64_to_2xi64_perm_mem_mask1:1904; CHECK: # %bb.0:1905; CHECK-NEXT: vmovdqa 16(%rdi), %xmm21906; CHECK-NEXT: vpblendd $12, (%rdi), %xmm2, %xmm2 # xmm2 = xmm2[0,1],mem[2,3]1907; CHECK-NEXT: vptestnmq %xmm1, %xmm1, %k11908; CHECK-NEXT: vmovdqa64 %xmm2, %xmm0 {%k1}1909; CHECK-NEXT: retq1910 %vec = load <4 x i64>, ptr %vp1911 %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 2, i32 1>1912 %cmp = icmp eq <2 x i64> %mask, zeroinitializer1913 %res = select <2 x i1> %cmp, <2 x i64> %shuf, <2 x i64> %vec21914 ret <2 x i64> %res1915}1916 1917define <2 x i64> @test_masked_z_4xi64_to_2xi64_perm_mem_mask1(ptr %vp, <2 x i64> %mask) {1918; CHECK-LABEL: test_masked_z_4xi64_to_2xi64_perm_mem_mask1:1919; CHECK: # %bb.0:1920; CHECK-NEXT: vmovdqa 16(%rdi), %xmm11921; CHECK-NEXT: vpblendd $12, (%rdi), %xmm1, %xmm1 # xmm1 = xmm1[0,1],mem[2,3]1922; CHECK-NEXT: vptestnmq %xmm0, %xmm0, %k11923; CHECK-NEXT: vmovdqa64 %xmm1, %xmm0 {%k1} {z}1924; CHECK-NEXT: retq1925 %vec = load <4 x i64>, ptr %vp1926 %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 2, i32 1>1927 %cmp = icmp eq <2 x i64> %mask, zeroinitializer1928 %res = select <2 x i1> %cmp, <2 x i64> %shuf, <2 x i64> zeroinitializer1929 ret <2 x i64> %res1930}1931 1932define <4 x i64> @test_8xi64_to_4xi64_perm_mask0(<8 x i64> %vec) {1933; CHECK-LABEL: test_8xi64_to_4xi64_perm_mask0:1934; CHECK: # %bb.0:1935; CHECK-NEXT: vextractf64x4 $1, %zmm0, %ymm01936; CHECK-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,3,2,1]1937; CHECK-NEXT: retq1938 %res = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 7, i32 6, i32 5>1939 ret <4 x i64> %res1940}1941define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask0(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {1942; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mask0:1943; CHECK: # %bb.0:1944; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm01945; CHECK-NEXT: vptestnmq %ymm2, %ymm2, %k11946; CHECK-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,3,2,1]1947; CHECK-NEXT: vmovdqa %ymm1, %ymm01948; CHECK-NEXT: retq1949 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 7, i32 6, i32 5>1950 %cmp = icmp eq <4 x i64> %mask, zeroinitializer1951 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec21952 ret <4 x i64> %res1953}1954 1955define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask0(<8 x i64> %vec, <4 x i64> %mask) {1956; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask0:1957; CHECK: # %bb.0:1958; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm01959; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k11960; CHECK-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3,2,1]1961; CHECK-NEXT: retq1962 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 7, i32 6, i32 5>1963 %cmp = icmp eq <4 x i64> %mask, zeroinitializer1964 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer1965 ret <4 x i64> %res1966}1967define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask1(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {1968; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mask1:1969; CHECK-FAST: # %bb.0:1970; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm11971; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [6,4,6,1]1972; CHECK-FAST-NEXT: vptestnmq %ymm2, %ymm2, %k11973; CHECK-FAST-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}1974; CHECK-FAST-NEXT: vmovdqa %ymm1, %ymm01975; CHECK-FAST-NEXT: retq1976;1977; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mask1:1978; CHECK-FAST-PERLANE: # %bb.0:1979; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm31980; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3],ymm3[4,5,6,7]1981; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm2, %ymm2, %k11982; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,0,2,1]1983; CHECK-FAST-PERLANE-NEXT: vmovdqa %ymm1, %ymm01984; CHECK-FAST-PERLANE-NEXT: retq1985 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 4, i32 6, i32 1>1986 %cmp = icmp eq <4 x i64> %mask, zeroinitializer1987 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec21988 ret <4 x i64> %res1989}1990 1991define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask1(<8 x i64> %vec, <4 x i64> %mask) {1992; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask1:1993; CHECK-FAST: # %bb.0:1994; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [6,4,6,1]1995; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k11996; CHECK-FAST-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}1997; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm01998; CHECK-FAST-NEXT: retq1999;2000; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask1:2001; CHECK-FAST-PERLANE: # %bb.0:2002; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm22003; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5,6,7]2004; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k12005; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,0,2,1]2006; CHECK-FAST-PERLANE-NEXT: retq2007 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 4, i32 6, i32 1>2008 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2009 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer2010 ret <4 x i64> %res2011}2012define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask2(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {2013; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mask2:2014; CHECK-FAST: # %bb.0:2015; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm12016; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [6,3,6,3]2017; CHECK-FAST-NEXT: vptestnmq %ymm2, %ymm2, %k12018; CHECK-FAST-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}2019; CHECK-FAST-NEXT: vmovdqa %ymm1, %ymm02020; CHECK-FAST-NEXT: retq2021;2022; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mask2:2023; CHECK-FAST-PERLANE: # %bb.0:2024; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm32025; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm0[2,3],ymm3[4,5],ymm0[6,7]2026; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm2, %ymm2, %k12027; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,3,2,3]2028; CHECK-FAST-PERLANE-NEXT: vmovdqa %ymm1, %ymm02029; CHECK-FAST-PERLANE-NEXT: retq2030 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 3, i32 6, i32 3>2031 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2032 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec22033 ret <4 x i64> %res2034}2035 2036define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask2(<8 x i64> %vec, <4 x i64> %mask) {2037; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask2:2038; CHECK-FAST: # %bb.0:2039; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [6,3,6,3]2040; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k12041; CHECK-FAST-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}2042; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02043; CHECK-FAST-NEXT: retq2044;2045; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask2:2046; CHECK-FAST-PERLANE: # %bb.0:2047; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm22048; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5],ymm0[6,7]2049; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k12050; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3,2,3]2051; CHECK-FAST-PERLANE-NEXT: retq2052 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 3, i32 6, i32 3>2053 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2054 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer2055 ret <4 x i64> %res2056}2057define <4 x i64> @test_8xi64_to_4xi64_perm_mask3(<8 x i64> %vec) {2058; CHECK-FAST-LABEL: test_8xi64_to_4xi64_perm_mask3:2059; CHECK-FAST: # %bb.0:2060; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [6,0,0,7]2061; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm1, %zmm02062; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02063; CHECK-FAST-NEXT: retq2064;2065; CHECK-FAST-PERLANE-LABEL: test_8xi64_to_4xi64_perm_mask3:2066; CHECK-FAST-PERLANE: # %bb.0:2067; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm12068; CHECK-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]2069; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,0,0,3]2070; CHECK-FAST-PERLANE-NEXT: retq2071 %res = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 0, i32 0, i32 7>2072 ret <4 x i64> %res2073}2074define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask3(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {2075; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mask3:2076; CHECK-FAST: # %bb.0:2077; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm12078; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [6,0,0,7]2079; CHECK-FAST-NEXT: vptestnmq %ymm2, %ymm2, %k12080; CHECK-FAST-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}2081; CHECK-FAST-NEXT: vmovdqa %ymm1, %ymm02082; CHECK-FAST-NEXT: retq2083;2084; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mask3:2085; CHECK-FAST-PERLANE: # %bb.0:2086; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm32087; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]2088; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm2, %ymm2, %k12089; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,0,0,3]2090; CHECK-FAST-PERLANE-NEXT: vmovdqa %ymm1, %ymm02091; CHECK-FAST-PERLANE-NEXT: retq2092 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 0, i32 0, i32 7>2093 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2094 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec22095 ret <4 x i64> %res2096}2097 2098define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask3(<8 x i64> %vec, <4 x i64> %mask) {2099; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask3:2100; CHECK-FAST: # %bb.0:2101; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [6,0,0,7]2102; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k12103; CHECK-FAST-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}2104; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02105; CHECK-FAST-NEXT: retq2106;2107; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask3:2108; CHECK-FAST-PERLANE: # %bb.0:2109; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm22110; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]2111; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k12112; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,0,0,3]2113; CHECK-FAST-PERLANE-NEXT: retq2114 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 6, i32 0, i32 0, i32 7>2115 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2116 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer2117 ret <4 x i64> %res2118}2119define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask4(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {2120; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mask4:2121; CHECK-FAST: # %bb.0:2122; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm12123; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [3,7,7,5]2124; CHECK-FAST-NEXT: vptestnmq %ymm2, %ymm2, %k12125; CHECK-FAST-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}2126; CHECK-FAST-NEXT: vmovdqa %ymm1, %ymm02127; CHECK-FAST-NEXT: retq2128;2129; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mask4:2130; CHECK-FAST-PERLANE: # %bb.0:2131; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm32132; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],ymm3[1],ymm0[3],ymm3[3]2133; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm2, %ymm2, %k12134; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,3,3,1]2135; CHECK-FAST-PERLANE-NEXT: vmovdqa %ymm1, %ymm02136; CHECK-FAST-PERLANE-NEXT: retq2137 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 3, i32 7, i32 7, i32 5>2138 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2139 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec22140 ret <4 x i64> %res2141}2142 2143define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask4(<8 x i64> %vec, <4 x i64> %mask) {2144; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask4:2145; CHECK-FAST: # %bb.0:2146; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [3,7,7,5]2147; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k12148; CHECK-FAST-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}2149; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02150; CHECK-FAST-NEXT: retq2151;2152; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask4:2153; CHECK-FAST-PERLANE: # %bb.0:2154; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm22155; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],ymm2[1],ymm0[3],ymm2[3]2156; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k12157; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3,3,1]2158; CHECK-FAST-PERLANE-NEXT: retq2159 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 3, i32 7, i32 7, i32 5>2160 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2161 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer2162 ret <4 x i64> %res2163}2164define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask5(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {2165; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mask5:2166; CHECK: # %bb.0:2167; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm12168; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [4,1,0,6]2169; CHECK-NEXT: vptestnmq %ymm2, %ymm2, %k12170; CHECK-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}2171; CHECK-NEXT: vmovdqa %ymm1, %ymm02172; CHECK-NEXT: retq2173 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 4, i32 1, i32 0, i32 6>2174 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2175 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec22176 ret <4 x i64> %res2177}2178 2179define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask5(<8 x i64> %vec, <4 x i64> %mask) {2180; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask5:2181; CHECK: # %bb.0:2182; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [4,1,0,6]2183; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k12184; CHECK-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}2185; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02186; CHECK-NEXT: retq2187 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 4, i32 1, i32 0, i32 6>2188 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2189 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer2190 ret <4 x i64> %res2191}2192define <4 x i64> @test_8xi64_to_4xi64_perm_mask6(<8 x i64> %vec) {2193; CHECK-FAST-LABEL: test_8xi64_to_4xi64_perm_mask6:2194; CHECK-FAST: # %bb.0:2195; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,6,5,3]2196; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm1, %zmm02197; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02198; CHECK-FAST-NEXT: retq2199;2200; CHECK-FAST-PERLANE-LABEL: test_8xi64_to_4xi64_perm_mask6:2201; CHECK-FAST-PERLANE: # %bb.0:2202; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm12203; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[3,2,1,3]2204; CHECK-FAST-PERLANE-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]2205; CHECK-FAST-PERLANE-NEXT: retq2206 %res = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 6, i32 5, i32 3>2207 ret <4 x i64> %res2208}2209define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask6(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {2210; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mask6:2211; CHECK-FAST: # %bb.0:2212; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm12213; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [7,6,5,3]2214; CHECK-FAST-NEXT: vptestnmq %ymm2, %ymm2, %k12215; CHECK-FAST-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}2216; CHECK-FAST-NEXT: vmovdqa %ymm1, %ymm02217; CHECK-FAST-NEXT: retq2218;2219; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mask6:2220; CHECK-FAST-PERLANE: # %bb.0:2221; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm32222; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm3 = ymm3[3,2,1,3]2223; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]2224; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm2, %ymm2, %k12225; CHECK-FAST-PERLANE-NEXT: vpblendmq %ymm0, %ymm1, %ymm0 {%k1}2226; CHECK-FAST-PERLANE-NEXT: retq2227 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 6, i32 5, i32 3>2228 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2229 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec22230 ret <4 x i64> %res2231}2232 2233define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask6(<8 x i64> %vec, <4 x i64> %mask) {2234; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask6:2235; CHECK-FAST: # %bb.0:2236; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,6,5,3]2237; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k12238; CHECK-FAST-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}2239; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02240; CHECK-FAST-NEXT: retq2241;2242; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask6:2243; CHECK-FAST-PERLANE: # %bb.0:2244; CHECK-FAST-PERLANE-NEXT: vextracti64x4 $1, %zmm0, %ymm22245; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm2 = ymm2[3,2,1,3]2246; CHECK-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]2247; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k12248; CHECK-FAST-PERLANE-NEXT: vmovdqa64 %ymm0, %ymm0 {%k1} {z}2249; CHECK-FAST-PERLANE-NEXT: retq2250 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 6, i32 5, i32 3>2251 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2252 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer2253 ret <4 x i64> %res2254}2255define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask7(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {2256; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mask7:2257; CHECK: # %bb.0:2258; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm12259; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [2,0,3,4]2260; CHECK-NEXT: vptestnmq %ymm2, %ymm2, %k12261; CHECK-NEXT: vpermq %zmm0, %zmm3, %zmm1 {%k1}2262; CHECK-NEXT: vmovdqa %ymm1, %ymm02263; CHECK-NEXT: retq2264 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 2, i32 0, i32 3, i32 4>2265 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2266 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec22267 ret <4 x i64> %res2268}2269define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask7(<8 x i64> %vec, <4 x i64> %mask) {2270; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask7:2271; CHECK: # %bb.0:2272; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [2,0,3,4]2273; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k12274; CHECK-NEXT: vpermq %zmm0, %zmm2, %zmm0 {%k1} {z}2275; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02276; CHECK-NEXT: retq2277 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 2, i32 0, i32 3, i32 4>2278 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2279 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer2280 ret <4 x i64> %res2281}2282 2283define <2 x i64> @test_8xi64_to_2xi64_perm_mask0(<8 x i64> %vec) {2284; CHECK-LABEL: test_8xi64_to_2xi64_perm_mask0:2285; CHECK: # %bb.0:2286; CHECK-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[3,0,2,3]2287; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm02288; CHECK-NEXT: vzeroupper2289; CHECK-NEXT: retq2290 %res = shufflevector <8 x i64> %vec, <8 x i64> undef, <2 x i32> <i32 3, i32 0>2291 ret <2 x i64> %res2292}2293define <2 x i64> @test_masked_8xi64_to_2xi64_perm_mask0(<8 x i64> %vec, <2 x i64> %vec2, <2 x i64> %mask) {2294; CHECK-LABEL: test_masked_8xi64_to_2xi64_perm_mask0:2295; CHECK: # %bb.0:2296; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm12297; CHECK-NEXT: vptestnmq %xmm2, %xmm2, %k12298; CHECK-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[3,0,2,3]2299; CHECK-NEXT: vmovdqa %xmm1, %xmm02300; CHECK-NEXT: vzeroupper2301; CHECK-NEXT: retq2302 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <2 x i32> <i32 3, i32 0>2303 %cmp = icmp eq <2 x i64> %mask, zeroinitializer2304 %res = select <2 x i1> %cmp, <2 x i64> %shuf, <2 x i64> %vec22305 ret <2 x i64> %res2306}2307define <2 x i64> @test_masked_z_8xi64_to_2xi64_perm_mask0(<8 x i64> %vec, <2 x i64> %mask) {2308; CHECK-LABEL: test_masked_z_8xi64_to_2xi64_perm_mask0:2309; CHECK: # %bb.0:2310; CHECK-NEXT: vptestnmq %xmm1, %xmm1, %k12311; CHECK-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[3,0,2,3]2312; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm02313; CHECK-NEXT: vzeroupper2314; CHECK-NEXT: retq2315 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <2 x i32> <i32 3, i32 0>2316 %cmp = icmp eq <2 x i64> %mask, zeroinitializer2317 %res = select <2 x i1> %cmp, <2 x i64> %shuf, <2 x i64> zeroinitializer2318 ret <2 x i64> %res2319}2320 2321define <2 x i64> @test_masked_8xi64_to_2xi64_perm_mask1(<8 x i64> %vec, <2 x i64> %vec2, <2 x i64> %mask) {2322; CHECK-LABEL: test_masked_8xi64_to_2xi64_perm_mask1:2323; CHECK: # %bb.0:2324; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm12325; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm02326; CHECK-NEXT: vptestnmq %xmm2, %xmm2, %k12327; CHECK-NEXT: vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,1,2,3]2328; CHECK-NEXT: vmovdqa %xmm1, %xmm02329; CHECK-NEXT: vzeroupper2330; CHECK-NEXT: retq2331 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <2 x i32> <i32 6, i32 5>2332 %cmp = icmp eq <2 x i64> %mask, zeroinitializer2333 %res = select <2 x i1> %cmp, <2 x i64> %shuf, <2 x i64> %vec22334 ret <2 x i64> %res2335}2336define <2 x i64> @test_masked_z_8xi64_to_2xi64_perm_mask1(<8 x i64> %vec, <2 x i64> %mask) {2337; CHECK-LABEL: test_masked_z_8xi64_to_2xi64_perm_mask1:2338; CHECK: # %bb.0:2339; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm02340; CHECK-NEXT: vptestnmq %xmm1, %xmm1, %k12341; CHECK-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,1,2,3]2342; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm02343; CHECK-NEXT: vzeroupper2344; CHECK-NEXT: retq2345 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <2 x i32> <i32 6, i32 5>2346 %cmp = icmp eq <2 x i64> %mask, zeroinitializer2347 %res = select <2 x i1> %cmp, <2 x i64> %shuf, <2 x i64> zeroinitializer2348 ret <2 x i64> %res2349}2350 2351define <4 x i64> @test_8xi64_to_4xi64_perm_mem_mask0(ptr %vp) {2352; CHECK-LABEL: test_8xi64_to_4xi64_perm_mem_mask0:2353; CHECK: # %bb.0:2354; CHECK-NEXT: vpermpd $136, (%rdi), %ymm0 # ymm0 = mem[0,2,0,2]2355; CHECK-NEXT: retq2356 %vec = load <8 x i64>, ptr %vp2357 %res = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 0, i32 2, i32 0, i32 2>2358 ret <4 x i64> %res2359}2360define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask0(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {2361; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask0:2362; CHECK: # %bb.0:2363; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k12364; CHECK-NEXT: vpermq $136, (%rdi), %ymm0 {%k1} # ymm0 {%k1} = mem[0,2,0,2]2365; CHECK-NEXT: retq2366 %vec = load <8 x i64>, ptr %vp2367 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 0, i32 2, i32 0, i32 2>2368 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2369 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec22370 ret <4 x i64> %res2371}2372define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask0(ptr %vp, <4 x i64> %mask) {2373; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask0:2374; CHECK: # %bb.0:2375; CHECK-NEXT: vptestnmq %ymm0, %ymm0, %k12376; CHECK-NEXT: vpermq $136, (%rdi), %ymm0 {%k1} {z} # ymm0 {%k1} {z} = mem[0,2,0,2]2377; CHECK-NEXT: retq2378 %vec = load <8 x i64>, ptr %vp2379 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 0, i32 2, i32 0, i32 2>2380 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2381 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer2382 ret <4 x i64> %res2383}2384 2385define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask1(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {2386; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask1:2387; CHECK-FAST: # %bb.0:2388; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm02389; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,7,6,0]2390; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k12391; CHECK-FAST-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}2392; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02393; CHECK-FAST-NEXT: retq2394;2395; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask1:2396; CHECK-FAST-PERLANE: # %bb.0:2397; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm22398; CHECK-FAST-PERLANE-NEXT: vpblendd $15, (%rdi), %ymm2, %ymm2 # ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]2399; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k12400; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} = ymm2[0,3,2,0]2401; CHECK-FAST-PERLANE-NEXT: retq2402 %vec = load <8 x i64>, ptr %vp2403 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 0, i32 7, i32 6, i32 0>2404 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2405 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec22406 ret <4 x i64> %res2407}2408 2409define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask1(ptr %vp, <4 x i64> %mask) {2410; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask1:2411; CHECK-FAST: # %bb.0:2412; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,7,6,0]2413; CHECK-FAST-NEXT: vptestnmq %ymm0, %ymm0, %k12414; CHECK-FAST-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}2415; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02416; CHECK-FAST-NEXT: retq2417;2418; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask1:2419; CHECK-FAST-PERLANE: # %bb.0:2420; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm12421; CHECK-FAST-PERLANE-NEXT: vpblendd $15, (%rdi), %ymm1, %ymm1 # ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]2422; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm0, %ymm0, %k12423; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm1[0,3,2,0]2424; CHECK-FAST-PERLANE-NEXT: retq2425 %vec = load <8 x i64>, ptr %vp2426 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 0, i32 7, i32 6, i32 0>2427 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2428 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer2429 ret <4 x i64> %res2430}2431 2432define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask2(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {2433; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask2:2434; CHECK-FAST: # %bb.0:2435; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm02436; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,1,1,5]2437; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k12438; CHECK-FAST-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}2439; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02440; CHECK-FAST-NEXT: retq2441;2442; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask2:2443; CHECK-FAST-PERLANE: # %bb.0:2444; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm22445; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq (%rdi), %ymm2, %ymm2 # ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]2446; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k12447; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} = ymm2[2,1,1,0]2448; CHECK-FAST-PERLANE-NEXT: retq2449 %vec = load <8 x i64>, ptr %vp2450 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 1, i32 1, i32 5>2451 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2452 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec22453 ret <4 x i64> %res2454}2455 2456define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask2(ptr %vp, <4 x i64> %mask) {2457; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask2:2458; CHECK-FAST: # %bb.0:2459; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,1,1,5]2460; CHECK-FAST-NEXT: vptestnmq %ymm0, %ymm0, %k12461; CHECK-FAST-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}2462; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02463; CHECK-FAST-NEXT: retq2464;2465; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask2:2466; CHECK-FAST-PERLANE: # %bb.0:2467; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm12468; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq (%rdi), %ymm1, %ymm1 # ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]2469; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm0, %ymm0, %k12470; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm1[2,1,1,0]2471; CHECK-FAST-PERLANE-NEXT: retq2472 %vec = load <8 x i64>, ptr %vp2473 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 1, i32 1, i32 5>2474 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2475 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer2476 ret <4 x i64> %res2477}2478 2479define <4 x i64> @test_8xi64_to_4xi64_perm_mem_mask3(ptr %vp) {2480; CHECK-FAST-LABEL: test_8xi64_to_4xi64_perm_mem_mask3:2481; CHECK-FAST: # %bb.0:2482; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm0 = [7,0,0,2]2483; CHECK-FAST-NEXT: vpermpd (%rdi), %zmm0, %zmm02484; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02485; CHECK-FAST-NEXT: retq2486;2487; CHECK-FAST-PERLANE-LABEL: test_8xi64_to_4xi64_perm_mem_mask3:2488; CHECK-FAST-PERLANE: # %bb.0:2489; CHECK-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm02490; CHECK-FAST-PERLANE-NEXT: vpalignr $8, 32(%rdi), %ymm0, %ymm0 # ymm0 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]2491; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,1,1,3]2492; CHECK-FAST-PERLANE-NEXT: retq2493 %vec = load <8 x i64>, ptr %vp2494 %res = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 0, i32 0, i32 2>2495 ret <4 x i64> %res2496}2497define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask3(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {2498; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask3:2499; CHECK-FAST: # %bb.0:2500; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm02501; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,0,0,2]2502; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k12503; CHECK-FAST-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}2504; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02505; CHECK-FAST-NEXT: retq2506;2507; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask3:2508; CHECK-FAST-PERLANE: # %bb.0:2509; CHECK-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm22510; CHECK-FAST-PERLANE-NEXT: vpalignr $8, 32(%rdi), %ymm2, %ymm2 # ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]2511; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k12512; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} = ymm2[2,1,1,3]2513; CHECK-FAST-PERLANE-NEXT: retq2514 %vec = load <8 x i64>, ptr %vp2515 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 0, i32 0, i32 2>2516 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2517 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec22518 ret <4 x i64> %res2519}2520 2521define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask3(ptr %vp, <4 x i64> %mask) {2522; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask3:2523; CHECK-FAST: # %bb.0:2524; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,0,0,2]2525; CHECK-FAST-NEXT: vptestnmq %ymm0, %ymm0, %k12526; CHECK-FAST-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}2527; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02528; CHECK-FAST-NEXT: retq2529;2530; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask3:2531; CHECK-FAST-PERLANE: # %bb.0:2532; CHECK-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm12533; CHECK-FAST-PERLANE-NEXT: vpalignr $8, 32(%rdi), %ymm1, %ymm1 # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]2534; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm0, %ymm0, %k12535; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm1[2,1,1,3]2536; CHECK-FAST-PERLANE-NEXT: retq2537 %vec = load <8 x i64>, ptr %vp2538 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 0, i32 0, i32 2>2539 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2540 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer2541 ret <4 x i64> %res2542}2543 2544define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask4(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {2545; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask4:2546; CHECK: # %bb.0:2547; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm02548; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,4,6,1]2549; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k12550; CHECK-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}2551; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02552; CHECK-NEXT: retq2553 %vec = load <8 x i64>, ptr %vp2554 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 0, i32 4, i32 6, i32 1>2555 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2556 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec22557 ret <4 x i64> %res2558}2559 2560define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask4(ptr %vp, <4 x i64> %mask) {2561; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask4:2562; CHECK: # %bb.0:2563; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,4,6,1]2564; CHECK-NEXT: vptestnmq %ymm0, %ymm0, %k12565; CHECK-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}2566; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02567; CHECK-NEXT: retq2568 %vec = load <8 x i64>, ptr %vp2569 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 0, i32 4, i32 6, i32 1>2570 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2571 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer2572 ret <4 x i64> %res2573}2574 2575define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask5(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {2576; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask5:2577; CHECK-FAST: # %bb.0:2578; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm02579; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,2,7,1]2580; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k12581; CHECK-FAST-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}2582; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02583; CHECK-FAST-NEXT: retq2584;2585; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask5:2586; CHECK-FAST-PERLANE: # %bb.0:2587; CHECK-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm22588; CHECK-FAST-PERLANE-NEXT: vpblendd $192, 32(%rdi), %ymm2, %ymm2 # ymm2 = ymm2[0,1,2,3,4,5],mem[6,7]2589; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k12590; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} = ymm2[0,2,3,1]2591; CHECK-FAST-PERLANE-NEXT: retq2592 %vec = load <8 x i64>, ptr %vp2593 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 0, i32 2, i32 7, i32 1>2594 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2595 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec22596 ret <4 x i64> %res2597}2598 2599define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask5(ptr %vp, <4 x i64> %mask) {2600; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask5:2601; CHECK-FAST: # %bb.0:2602; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,7,1]2603; CHECK-FAST-NEXT: vptestnmq %ymm0, %ymm0, %k12604; CHECK-FAST-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}2605; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02606; CHECK-FAST-NEXT: retq2607;2608; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask5:2609; CHECK-FAST-PERLANE: # %bb.0:2610; CHECK-FAST-PERLANE-NEXT: vmovdqa (%rdi), %ymm12611; CHECK-FAST-PERLANE-NEXT: vpblendd $192, 32(%rdi), %ymm1, %ymm1 # ymm1 = ymm1[0,1,2,3,4,5],mem[6,7]2612; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm0, %ymm0, %k12613; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm1[0,2,3,1]2614; CHECK-FAST-PERLANE-NEXT: retq2615 %vec = load <8 x i64>, ptr %vp2616 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 0, i32 2, i32 7, i32 1>2617 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2618 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer2619 ret <4 x i64> %res2620}2621 2622define <4 x i64> @test_8xi64_to_4xi64_perm_mem_mask6(ptr %vp) {2623; CHECK-LABEL: test_8xi64_to_4xi64_perm_mem_mask6:2624; CHECK: # %bb.0:2625; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm0 = [7,2,3,2]2626; CHECK-NEXT: vpermpd (%rdi), %zmm0, %zmm02627; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02628; CHECK-NEXT: retq2629 %vec = load <8 x i64>, ptr %vp2630 %res = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 2, i32 3, i32 2>2631 ret <4 x i64> %res2632}2633define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask6(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {2634; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask6:2635; CHECK: # %bb.0:2636; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm02637; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,2,3,2]2638; CHECK-NEXT: vptestnmq %ymm1, %ymm1, %k12639; CHECK-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}2640; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02641; CHECK-NEXT: retq2642 %vec = load <8 x i64>, ptr %vp2643 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 2, i32 3, i32 2>2644 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2645 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec22646 ret <4 x i64> %res2647}2648 2649define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask6(ptr %vp, <4 x i64> %mask) {2650; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask6:2651; CHECK: # %bb.0:2652; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,2,3,2]2653; CHECK-NEXT: vptestnmq %ymm0, %ymm0, %k12654; CHECK-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}2655; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02656; CHECK-NEXT: retq2657 %vec = load <8 x i64>, ptr %vp2658 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 2, i32 3, i32 2>2659 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2660 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer2661 ret <4 x i64> %res2662}2663 2664define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mem_mask7(ptr %vp, <4 x i64> %vec2, <4 x i64> %mask) {2665; CHECK-FAST-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask7:2666; CHECK-FAST: # %bb.0:2667; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm02668; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,7,5,1]2669; CHECK-FAST-NEXT: vptestnmq %ymm1, %ymm1, %k12670; CHECK-FAST-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}2671; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02672; CHECK-FAST-NEXT: retq2673;2674; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_4xi64_perm_mem_mask7:2675; CHECK-FAST-PERLANE: # %bb.0:2676; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm22677; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq (%rdi), %ymm2, %ymm2 # ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]2678; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm1, %ymm1, %k12679; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} = ymm2[2,2,0,1]2680; CHECK-FAST-PERLANE-NEXT: retq2681 %vec = load <8 x i64>, ptr %vp2682 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 7, i32 5, i32 1>2683 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2684 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec22685 ret <4 x i64> %res2686}2687 2688define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mem_mask7(ptr %vp, <4 x i64> %mask) {2689; CHECK-FAST-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask7:2690; CHECK-FAST: # %bb.0:2691; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,7,5,1]2692; CHECK-FAST-NEXT: vptestnmq %ymm0, %ymm0, %k12693; CHECK-FAST-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}2694; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm02695; CHECK-FAST-NEXT: retq2696;2697; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_4xi64_perm_mem_mask7:2698; CHECK-FAST-PERLANE: # %bb.0:2699; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %ymm12700; CHECK-FAST-PERLANE-NEXT: vpunpckhqdq (%rdi), %ymm1, %ymm1 # ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]2701; CHECK-FAST-PERLANE-NEXT: vptestnmq %ymm0, %ymm0, %k12702; CHECK-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 {%k1} {z} = ymm1[2,2,0,1]2703; CHECK-FAST-PERLANE-NEXT: retq2704 %vec = load <8 x i64>, ptr %vp2705 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <4 x i32> <i32 7, i32 7, i32 5, i32 1>2706 %cmp = icmp eq <4 x i64> %mask, zeroinitializer2707 %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer2708 ret <4 x i64> %res2709}2710 2711define <2 x i64> @test_8xi64_to_2xi64_perm_mem_mask0(ptr %vp) {2712; CHECK-FAST-LABEL: test_8xi64_to_2xi64_perm_mem_mask0:2713; CHECK-FAST: # %bb.0:2714; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} xmm0 = [4,1]2715; CHECK-FAST-NEXT: vpermpd (%rdi), %zmm0, %zmm02716; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm02717; CHECK-FAST-NEXT: vzeroupper2718; CHECK-FAST-NEXT: retq2719;2720; CHECK-FAST-PERLANE-LABEL: test_8xi64_to_2xi64_perm_mem_mask0:2721; CHECK-FAST-PERLANE: # %bb.0:2722; CHECK-FAST-PERLANE-NEXT: vmovaps 32(%rdi), %xmm02723; CHECK-FAST-PERLANE-NEXT: vblendps $12, (%rdi), %xmm0, %xmm0 # xmm0 = xmm0[0,1],mem[2,3]2724; CHECK-FAST-PERLANE-NEXT: retq2725 %vec = load <8 x i64>, ptr %vp2726 %res = shufflevector <8 x i64> %vec, <8 x i64> undef, <2 x i32> <i32 4, i32 1>2727 ret <2 x i64> %res2728}2729define <2 x i64> @test_masked_8xi64_to_2xi64_perm_mem_mask0(ptr %vp, <2 x i64> %vec2, <2 x i64> %mask) {2730; CHECK-FAST-LABEL: test_masked_8xi64_to_2xi64_perm_mem_mask0:2731; CHECK-FAST: # %bb.0:2732; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm02733; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} xmm2 = [4,1]2734; CHECK-FAST-NEXT: vptestnmq %xmm1, %xmm1, %k12735; CHECK-FAST-NEXT: vpermq (%rdi), %zmm2, %zmm0 {%k1}2736; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm02737; CHECK-FAST-NEXT: vzeroupper2738; CHECK-FAST-NEXT: retq2739;2740; CHECK-FAST-PERLANE-LABEL: test_masked_8xi64_to_2xi64_perm_mem_mask0:2741; CHECK-FAST-PERLANE: # %bb.0:2742; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %xmm22743; CHECK-FAST-PERLANE-NEXT: vpblendd $12, (%rdi), %xmm2, %xmm2 # xmm2 = xmm2[0,1],mem[2,3]2744; CHECK-FAST-PERLANE-NEXT: vptestnmq %xmm1, %xmm1, %k12745; CHECK-FAST-PERLANE-NEXT: vmovdqa64 %xmm2, %xmm0 {%k1}2746; CHECK-FAST-PERLANE-NEXT: retq2747 %vec = load <8 x i64>, ptr %vp2748 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <2 x i32> <i32 4, i32 1>2749 %cmp = icmp eq <2 x i64> %mask, zeroinitializer2750 %res = select <2 x i1> %cmp, <2 x i64> %shuf, <2 x i64> %vec22751 ret <2 x i64> %res2752}2753 2754define <2 x i64> @test_masked_z_8xi64_to_2xi64_perm_mem_mask0(ptr %vp, <2 x i64> %mask) {2755; CHECK-FAST-LABEL: test_masked_z_8xi64_to_2xi64_perm_mem_mask0:2756; CHECK-FAST: # %bb.0:2757; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} xmm1 = [4,1]2758; CHECK-FAST-NEXT: vptestnmq %xmm0, %xmm0, %k12759; CHECK-FAST-NEXT: vpermq (%rdi), %zmm1, %zmm0 {%k1} {z}2760; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm02761; CHECK-FAST-NEXT: vzeroupper2762; CHECK-FAST-NEXT: retq2763;2764; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xi64_to_2xi64_perm_mem_mask0:2765; CHECK-FAST-PERLANE: # %bb.0:2766; CHECK-FAST-PERLANE-NEXT: vmovdqa 32(%rdi), %xmm12767; CHECK-FAST-PERLANE-NEXT: vpblendd $12, (%rdi), %xmm1, %xmm1 # xmm1 = xmm1[0,1],mem[2,3]2768; CHECK-FAST-PERLANE-NEXT: vptestnmq %xmm0, %xmm0, %k12769; CHECK-FAST-PERLANE-NEXT: vmovdqa64 %xmm1, %xmm0 {%k1} {z}2770; CHECK-FAST-PERLANE-NEXT: retq2771 %vec = load <8 x i64>, ptr %vp2772 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <2 x i32> <i32 4, i32 1>2773 %cmp = icmp eq <2 x i64> %mask, zeroinitializer2774 %res = select <2 x i1> %cmp, <2 x i64> %shuf, <2 x i64> zeroinitializer2775 ret <2 x i64> %res2776}2777 2778define <2 x i64> @test_masked_8xi64_to_2xi64_perm_mem_mask1(ptr %vp, <2 x i64> %vec2, <2 x i64> %mask) {2779; CHECK-LABEL: test_masked_8xi64_to_2xi64_perm_mem_mask1:2780; CHECK: # %bb.0:2781; CHECK-NEXT: vmovdqa 48(%rdi), %xmm22782; CHECK-NEXT: vptestnmq %xmm1, %xmm1, %k12783; CHECK-NEXT: vpunpcklqdq 16(%rdi), %xmm2, %xmm0 {%k1} # xmm0 {%k1} = xmm2[0],mem[0]2784; CHECK-NEXT: retq2785 %vec = load <8 x i64>, ptr %vp2786 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <2 x i32> <i32 6, i32 2>2787 %cmp = icmp eq <2 x i64> %mask, zeroinitializer2788 %res = select <2 x i1> %cmp, <2 x i64> %shuf, <2 x i64> %vec22789 ret <2 x i64> %res2790}2791 2792define <2 x i64> @test_masked_z_8xi64_to_2xi64_perm_mem_mask1(ptr %vp, <2 x i64> %mask) {2793; CHECK-LABEL: test_masked_z_8xi64_to_2xi64_perm_mem_mask1:2794; CHECK: # %bb.0:2795; CHECK-NEXT: vmovdqa 48(%rdi), %xmm12796; CHECK-NEXT: vptestnmq %xmm0, %xmm0, %k12797; CHECK-NEXT: vpunpcklqdq 16(%rdi), %xmm1, %xmm0 {%k1} {z} # xmm0 {%k1} {z} = xmm1[0],mem[0]2798; CHECK-NEXT: retq2799 %vec = load <8 x i64>, ptr %vp2800 %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <2 x i32> <i32 6, i32 2>2801 %cmp = icmp eq <2 x i64> %mask, zeroinitializer2802 %res = select <2 x i1> %cmp, <2 x i64> %shuf, <2 x i64> zeroinitializer2803 ret <2 x i64> %res2804}2805 2806define <4 x float> @test_8xfloat_to_4xfloat_perm_mask0(<8 x float> %vec) {2807; CHECK-LABEL: test_8xfloat_to_4xfloat_perm_mask0:2808; CHECK: # %bb.0:2809; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm12810; CHECK-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,3],xmm1[0,1]2811; CHECK-NEXT: vzeroupper2812; CHECK-NEXT: retq2813 %res = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 0, i32 3, i32 4, i32 5>2814 ret <4 x float> %res2815}2816define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mask0(<8 x float> %vec, <4 x float> %vec2, <4 x float> %mask) {2817; CHECK-LABEL: test_masked_8xfloat_to_4xfloat_perm_mask0:2818; CHECK: # %bb.0:2819; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm32820; CHECK-NEXT: vxorps %xmm4, %xmm4, %xmm42821; CHECK-NEXT: vcmpeqps %xmm4, %xmm2, %k12822; CHECK-NEXT: vshufps {{.*#+}} xmm1 {%k1} = xmm0[0,3],xmm3[0,1]2823; CHECK-NEXT: vmovaps %xmm1, %xmm02824; CHECK-NEXT: vzeroupper2825; CHECK-NEXT: retq2826 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 0, i32 3, i32 4, i32 5>2827 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer2828 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec22829 ret <4 x float> %res2830}2831 2832define <4 x float> @test_masked_z_8xfloat_to_4xfloat_perm_mask0(<8 x float> %vec, <4 x float> %mask) {2833; CHECK-LABEL: test_masked_z_8xfloat_to_4xfloat_perm_mask0:2834; CHECK: # %bb.0:2835; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm22836; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm32837; CHECK-NEXT: vcmpeqps %xmm3, %xmm1, %k12838; CHECK-NEXT: vshufps {{.*#+}} xmm0 {%k1} {z} = xmm0[0,3],xmm2[0,1]2839; CHECK-NEXT: vzeroupper2840; CHECK-NEXT: retq2841 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 0, i32 3, i32 4, i32 5>2842 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer2843 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer2844 ret <4 x float> %res2845}2846define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mask1(<8 x float> %vec, <4 x float> %vec2, <4 x float> %mask) {2847; CHECK-LABEL: test_masked_8xfloat_to_4xfloat_perm_mask1:2848; CHECK: # %bb.0:2849; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm12850; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm3 = [1,3,5,0]2851; CHECK-NEXT: vxorps %xmm4, %xmm4, %xmm42852; CHECK-NEXT: vcmpeqps %xmm4, %xmm2, %k12853; CHECK-NEXT: vpermps %ymm0, %ymm3, %ymm1 {%k1}2854; CHECK-NEXT: vmovaps %xmm1, %xmm02855; CHECK-NEXT: vzeroupper2856; CHECK-NEXT: retq2857 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 0>2858 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer2859 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec22860 ret <4 x float> %res2861}2862 2863define <4 x float> @test_masked_z_8xfloat_to_4xfloat_perm_mask1(<8 x float> %vec, <4 x float> %mask) {2864; CHECK-LABEL: test_masked_z_8xfloat_to_4xfloat_perm_mask1:2865; CHECK: # %bb.0:2866; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [1,3,5,0]2867; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm32868; CHECK-NEXT: vcmpeqps %xmm3, %xmm1, %k12869; CHECK-NEXT: vpermps %ymm0, %ymm2, %ymm0 {%k1} {z}2870; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm02871; CHECK-NEXT: vzeroupper2872; CHECK-NEXT: retq2873 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 0>2874 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer2875 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer2876 ret <4 x float> %res2877}2878define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mask2(<8 x float> %vec, <4 x float> %vec2, <4 x float> %mask) {2879; CHECK-LABEL: test_masked_8xfloat_to_4xfloat_perm_mask2:2880; CHECK: # %bb.0:2881; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm12882; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm3 = [3,2,7,0]2883; CHECK-NEXT: vxorps %xmm4, %xmm4, %xmm42884; CHECK-NEXT: vcmpeqps %xmm4, %xmm2, %k12885; CHECK-NEXT: vpermps %ymm0, %ymm3, %ymm1 {%k1}2886; CHECK-NEXT: vmovaps %xmm1, %xmm02887; CHECK-NEXT: vzeroupper2888; CHECK-NEXT: retq2889 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 3, i32 2, i32 7, i32 0>2890 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer2891 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec22892 ret <4 x float> %res2893}2894 2895define <4 x float> @test_masked_z_8xfloat_to_4xfloat_perm_mask2(<8 x float> %vec, <4 x float> %mask) {2896; CHECK-LABEL: test_masked_z_8xfloat_to_4xfloat_perm_mask2:2897; CHECK: # %bb.0:2898; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [3,2,7,0]2899; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm32900; CHECK-NEXT: vcmpeqps %xmm3, %xmm1, %k12901; CHECK-NEXT: vpermps %ymm0, %ymm2, %ymm0 {%k1} {z}2902; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm02903; CHECK-NEXT: vzeroupper2904; CHECK-NEXT: retq2905 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 3, i32 2, i32 7, i32 0>2906 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer2907 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer2908 ret <4 x float> %res2909}2910define <4 x float> @test_8xfloat_to_4xfloat_perm_mask3(<8 x float> %vec) {2911; CHECK-LABEL: test_8xfloat_to_4xfloat_perm_mask3:2912; CHECK: # %bb.0:2913; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [3,3,5,2]2914; CHECK-NEXT: vpermps %ymm0, %ymm1, %ymm02915; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm02916; CHECK-NEXT: vzeroupper2917; CHECK-NEXT: retq2918 %res = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 3, i32 3, i32 5, i32 2>2919 ret <4 x float> %res2920}2921define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mask3(<8 x float> %vec, <4 x float> %vec2, <4 x float> %mask) {2922; CHECK-LABEL: test_masked_8xfloat_to_4xfloat_perm_mask3:2923; CHECK: # %bb.0:2924; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm12925; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm3 = [3,3,5,2]2926; CHECK-NEXT: vxorps %xmm4, %xmm4, %xmm42927; CHECK-NEXT: vcmpeqps %xmm4, %xmm2, %k12928; CHECK-NEXT: vpermps %ymm0, %ymm3, %ymm1 {%k1}2929; CHECK-NEXT: vmovaps %xmm1, %xmm02930; CHECK-NEXT: vzeroupper2931; CHECK-NEXT: retq2932 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 3, i32 3, i32 5, i32 2>2933 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer2934 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec22935 ret <4 x float> %res2936}2937 2938define <4 x float> @test_masked_z_8xfloat_to_4xfloat_perm_mask3(<8 x float> %vec, <4 x float> %mask) {2939; CHECK-LABEL: test_masked_z_8xfloat_to_4xfloat_perm_mask3:2940; CHECK: # %bb.0:2941; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [3,3,5,2]2942; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm32943; CHECK-NEXT: vcmpeqps %xmm3, %xmm1, %k12944; CHECK-NEXT: vpermps %ymm0, %ymm2, %ymm0 {%k1} {z}2945; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm02946; CHECK-NEXT: vzeroupper2947; CHECK-NEXT: retq2948 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 3, i32 3, i32 5, i32 2>2949 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer2950 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer2951 ret <4 x float> %res2952}2953define <4 x float> @test_8xfloat_to_4xfloat_perm_mem_mask0(ptr %vp) {2954; CHECK-LABEL: test_8xfloat_to_4xfloat_perm_mem_mask0:2955; CHECK: # %bb.0:2956; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm0 = [6,2,4,5]2957; CHECK-NEXT: vpermps (%rdi), %ymm0, %ymm02958; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm02959; CHECK-NEXT: vzeroupper2960; CHECK-NEXT: retq2961 %vec = load <8 x float>, ptr %vp2962 %res = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 6, i32 2, i32 4, i32 5>2963 ret <4 x float> %res2964}2965define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mem_mask0(ptr %vp, <4 x float> %vec2, <4 x float> %mask) {2966; CHECK-LABEL: test_masked_8xfloat_to_4xfloat_perm_mem_mask0:2967; CHECK: # %bb.0:2968; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm02969; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [6,2,4,5]2970; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm32971; CHECK-NEXT: vcmpeqps %xmm3, %xmm1, %k12972; CHECK-NEXT: vpermps (%rdi), %ymm2, %ymm0 {%k1}2973; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm02974; CHECK-NEXT: vzeroupper2975; CHECK-NEXT: retq2976 %vec = load <8 x float>, ptr %vp2977 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 6, i32 2, i32 4, i32 5>2978 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer2979 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec22980 ret <4 x float> %res2981}2982 2983define <4 x float> @test_masked_z_8xfloat_to_4xfloat_perm_mem_mask0(ptr %vp, <4 x float> %mask) {2984; CHECK-LABEL: test_masked_z_8xfloat_to_4xfloat_perm_mem_mask0:2985; CHECK: # %bb.0:2986; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [6,2,4,5]2987; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm22988; CHECK-NEXT: vcmpeqps %xmm2, %xmm0, %k12989; CHECK-NEXT: vpermps (%rdi), %ymm1, %ymm0 {%k1} {z}2990; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm02991; CHECK-NEXT: vzeroupper2992; CHECK-NEXT: retq2993 %vec = load <8 x float>, ptr %vp2994 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 6, i32 2, i32 4, i32 5>2995 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer2996 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer2997 ret <4 x float> %res2998}2999 3000define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mem_mask1(ptr %vp, <4 x float> %vec2, <4 x float> %mask) {3001; CHECK-LABEL: test_masked_8xfloat_to_4xfloat_perm_mem_mask1:3002; CHECK: # %bb.0:3003; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm03004; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [6,3,3,6]3005; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm33006; CHECK-NEXT: vcmpeqps %xmm3, %xmm1, %k13007; CHECK-NEXT: vpermps (%rdi), %ymm2, %ymm0 {%k1}3008; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm03009; CHECK-NEXT: vzeroupper3010; CHECK-NEXT: retq3011 %vec = load <8 x float>, ptr %vp3012 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 6, i32 3, i32 3, i32 6>3013 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3014 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec23015 ret <4 x float> %res3016}3017 3018define <4 x float> @test_masked_z_8xfloat_to_4xfloat_perm_mem_mask1(ptr %vp, <4 x float> %mask) {3019; CHECK-LABEL: test_masked_z_8xfloat_to_4xfloat_perm_mem_mask1:3020; CHECK: # %bb.0:3021; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [6,3,3,6]3022; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm23023; CHECK-NEXT: vcmpeqps %xmm2, %xmm0, %k13024; CHECK-NEXT: vpermps (%rdi), %ymm1, %ymm0 {%k1} {z}3025; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm03026; CHECK-NEXT: vzeroupper3027; CHECK-NEXT: retq3028 %vec = load <8 x float>, ptr %vp3029 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 6, i32 3, i32 3, i32 6>3030 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3031 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer3032 ret <4 x float> %res3033}3034 3035define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mem_mask2(ptr %vp, <4 x float> %vec2, <4 x float> %mask) {3036; CHECK-LABEL: test_masked_8xfloat_to_4xfloat_perm_mem_mask2:3037; CHECK: # %bb.0:3038; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm03039; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [3,1,3,7]3040; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm33041; CHECK-NEXT: vcmpeqps %xmm3, %xmm1, %k13042; CHECK-NEXT: vpermps (%rdi), %ymm2, %ymm0 {%k1}3043; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm03044; CHECK-NEXT: vzeroupper3045; CHECK-NEXT: retq3046 %vec = load <8 x float>, ptr %vp3047 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 3, i32 1, i32 3, i32 7>3048 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3049 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec23050 ret <4 x float> %res3051}3052 3053define <4 x float> @test_masked_z_8xfloat_to_4xfloat_perm_mem_mask2(ptr %vp, <4 x float> %mask) {3054; CHECK-LABEL: test_masked_z_8xfloat_to_4xfloat_perm_mem_mask2:3055; CHECK: # %bb.0:3056; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [3,1,3,7]3057; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm23058; CHECK-NEXT: vcmpeqps %xmm2, %xmm0, %k13059; CHECK-NEXT: vpermps (%rdi), %ymm1, %ymm0 {%k1} {z}3060; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm03061; CHECK-NEXT: vzeroupper3062; CHECK-NEXT: retq3063 %vec = load <8 x float>, ptr %vp3064 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 3, i32 1, i32 3, i32 7>3065 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3066 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer3067 ret <4 x float> %res3068}3069 3070define <4 x float> @test_8xfloat_to_4xfloat_perm_mem_mask3(ptr %vp) {3071; CHECK-LABEL: test_8xfloat_to_4xfloat_perm_mem_mask3:3072; CHECK: # %bb.0:3073; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm0 = [1,3,5,3]3074; CHECK-NEXT: vpermps (%rdi), %ymm0, %ymm03075; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm03076; CHECK-NEXT: vzeroupper3077; CHECK-NEXT: retq3078 %vec = load <8 x float>, ptr %vp3079 %res = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 3>3080 ret <4 x float> %res3081}3082define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mem_mask3(ptr %vp, <4 x float> %vec2, <4 x float> %mask) {3083; CHECK-LABEL: test_masked_8xfloat_to_4xfloat_perm_mem_mask3:3084; CHECK: # %bb.0:3085; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm03086; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [1,3,5,3]3087; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm33088; CHECK-NEXT: vcmpeqps %xmm3, %xmm1, %k13089; CHECK-NEXT: vpermps (%rdi), %ymm2, %ymm0 {%k1}3090; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm03091; CHECK-NEXT: vzeroupper3092; CHECK-NEXT: retq3093 %vec = load <8 x float>, ptr %vp3094 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 3>3095 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3096 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec23097 ret <4 x float> %res3098}3099 3100define <4 x float> @test_masked_z_8xfloat_to_4xfloat_perm_mem_mask3(ptr %vp, <4 x float> %mask) {3101; CHECK-LABEL: test_masked_z_8xfloat_to_4xfloat_perm_mem_mask3:3102; CHECK: # %bb.0:3103; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [1,3,5,3]3104; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm23105; CHECK-NEXT: vcmpeqps %xmm2, %xmm0, %k13106; CHECK-NEXT: vpermps (%rdi), %ymm1, %ymm0 {%k1} {z}3107; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm03108; CHECK-NEXT: vzeroupper3109; CHECK-NEXT: retq3110 %vec = load <8 x float>, ptr %vp3111 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 3>3112 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3113 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer3114 ret <4 x float> %res3115}3116 3117define <8 x float> @test_16xfloat_to_8xfloat_perm_mask0(<16 x float> %vec) {3118; CHECK-LABEL: test_16xfloat_to_8xfloat_perm_mask0:3119; CHECK: # %bb.0:3120; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,4,12,10,8,2,11,7]3121; CHECK-NEXT: vpermps %zmm0, %zmm1, %zmm03122; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03123; CHECK-NEXT: retq3124 %res = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 0, i32 4, i32 12, i32 10, i32 8, i32 2, i32 11, i32 7>3125 ret <8 x float> %res3126}3127define <8 x float> @test_masked_16xfloat_to_8xfloat_perm_mask0(<16 x float> %vec, <8 x float> %vec2, <8 x float> %mask) {3128; CHECK-LABEL: test_masked_16xfloat_to_8xfloat_perm_mask0:3129; CHECK: # %bb.0:3130; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm13131; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,4,12,10,8,2,11,7]3132; CHECK-NEXT: vxorps %xmm4, %xmm4, %xmm43133; CHECK-NEXT: vcmpeqps %ymm4, %ymm2, %k13134; CHECK-NEXT: vpermps %zmm0, %zmm3, %zmm1 {%k1}3135; CHECK-NEXT: vmovaps %ymm1, %ymm03136; CHECK-NEXT: retq3137 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 0, i32 4, i32 12, i32 10, i32 8, i32 2, i32 11, i32 7>3138 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer3139 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec23140 ret <8 x float> %res3141}3142 3143define <8 x float> @test_masked_z_16xfloat_to_8xfloat_perm_mask0(<16 x float> %vec, <8 x float> %mask) {3144; CHECK-LABEL: test_masked_z_16xfloat_to_8xfloat_perm_mask0:3145; CHECK: # %bb.0:3146; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,4,12,10,8,2,11,7]3147; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm33148; CHECK-NEXT: vcmpeqps %ymm3, %ymm1, %k13149; CHECK-NEXT: vpermps %zmm0, %zmm2, %zmm0 {%k1} {z}3150; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03151; CHECK-NEXT: retq3152 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 0, i32 4, i32 12, i32 10, i32 8, i32 2, i32 11, i32 7>3153 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer3154 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer3155 ret <8 x float> %res3156}3157define <8 x float> @test_masked_16xfloat_to_8xfloat_perm_mask1(<16 x float> %vec, <8 x float> %vec2, <8 x float> %mask) {3158; CHECK-LABEL: test_masked_16xfloat_to_8xfloat_perm_mask1:3159; CHECK: # %bb.0:3160; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm13161; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm3 = [10,12,3,12,4,15,1,14]3162; CHECK-NEXT: vxorps %xmm4, %xmm4, %xmm43163; CHECK-NEXT: vcmpeqps %ymm4, %ymm2, %k13164; CHECK-NEXT: vpermps %zmm0, %zmm3, %zmm1 {%k1}3165; CHECK-NEXT: vmovaps %ymm1, %ymm03166; CHECK-NEXT: retq3167 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 10, i32 12, i32 3, i32 12, i32 4, i32 15, i32 1, i32 14>3168 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer3169 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec23170 ret <8 x float> %res3171}3172 3173define <8 x float> @test_masked_z_16xfloat_to_8xfloat_perm_mask1(<16 x float> %vec, <8 x float> %mask) {3174; CHECK-LABEL: test_masked_z_16xfloat_to_8xfloat_perm_mask1:3175; CHECK: # %bb.0:3176; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [10,12,3,12,4,15,1,14]3177; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm33178; CHECK-NEXT: vcmpeqps %ymm3, %ymm1, %k13179; CHECK-NEXT: vpermps %zmm0, %zmm2, %zmm0 {%k1} {z}3180; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03181; CHECK-NEXT: retq3182 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 10, i32 12, i32 3, i32 12, i32 4, i32 15, i32 1, i32 14>3183 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer3184 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer3185 ret <8 x float> %res3186}3187define <8 x float> @test_masked_16xfloat_to_8xfloat_perm_mask2(<16 x float> %vec, <8 x float> %vec2, <8 x float> %mask) {3188; CHECK-LABEL: test_masked_16xfloat_to_8xfloat_perm_mask2:3189; CHECK: # %bb.0:3190; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm13191; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,4,8,9,6,1,4,4]3192; CHECK-NEXT: vxorps %xmm4, %xmm4, %xmm43193; CHECK-NEXT: vcmpeqps %ymm4, %ymm2, %k13194; CHECK-NEXT: vpermps %zmm0, %zmm3, %zmm1 {%k1}3195; CHECK-NEXT: vmovaps %ymm1, %ymm03196; CHECK-NEXT: retq3197 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 9, i32 6, i32 1, i32 4, i32 4>3198 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer3199 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec23200 ret <8 x float> %res3201}3202 3203define <8 x float> @test_masked_z_16xfloat_to_8xfloat_perm_mask2(<16 x float> %vec, <8 x float> %mask) {3204; CHECK-LABEL: test_masked_z_16xfloat_to_8xfloat_perm_mask2:3205; CHECK: # %bb.0:3206; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,4,8,9,6,1,4,4]3207; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm33208; CHECK-NEXT: vcmpeqps %ymm3, %ymm1, %k13209; CHECK-NEXT: vpermps %zmm0, %zmm2, %zmm0 {%k1} {z}3210; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03211; CHECK-NEXT: retq3212 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 9, i32 6, i32 1, i32 4, i32 4>3213 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer3214 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer3215 ret <8 x float> %res3216}3217define <8 x float> @test_16xfloat_to_8xfloat_perm_mask3(<16 x float> %vec) {3218; CHECK-LABEL: test_16xfloat_to_8xfloat_perm_mask3:3219; CHECK: # %bb.0:3220; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [12,14,9,0,12,4,5,8]3221; CHECK-NEXT: vpermps %zmm0, %zmm1, %zmm03222; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03223; CHECK-NEXT: retq3224 %res = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 12, i32 14, i32 9, i32 0, i32 12, i32 4, i32 5, i32 8>3225 ret <8 x float> %res3226}3227define <8 x float> @test_masked_16xfloat_to_8xfloat_perm_mask3(<16 x float> %vec, <8 x float> %vec2, <8 x float> %mask) {3228; CHECK-LABEL: test_masked_16xfloat_to_8xfloat_perm_mask3:3229; CHECK: # %bb.0:3230; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm13231; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm3 = [12,14,9,0,12,4,5,8]3232; CHECK-NEXT: vxorps %xmm4, %xmm4, %xmm43233; CHECK-NEXT: vcmpeqps %ymm4, %ymm2, %k13234; CHECK-NEXT: vpermps %zmm0, %zmm3, %zmm1 {%k1}3235; CHECK-NEXT: vmovaps %ymm1, %ymm03236; CHECK-NEXT: retq3237 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 12, i32 14, i32 9, i32 0, i32 12, i32 4, i32 5, i32 8>3238 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer3239 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec23240 ret <8 x float> %res3241}3242 3243define <8 x float> @test_masked_z_16xfloat_to_8xfloat_perm_mask3(<16 x float> %vec, <8 x float> %mask) {3244; CHECK-LABEL: test_masked_z_16xfloat_to_8xfloat_perm_mask3:3245; CHECK: # %bb.0:3246; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [12,14,9,0,12,4,5,8]3247; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm33248; CHECK-NEXT: vcmpeqps %ymm3, %ymm1, %k13249; CHECK-NEXT: vpermps %zmm0, %zmm2, %zmm0 {%k1} {z}3250; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03251; CHECK-NEXT: retq3252 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 12, i32 14, i32 9, i32 0, i32 12, i32 4, i32 5, i32 8>3253 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer3254 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer3255 ret <8 x float> %res3256}3257define <4 x float> @test_16xfloat_to_4xfloat_perm_mask0(<16 x float> %vec) {3258; CHECK-LABEL: test_16xfloat_to_4xfloat_perm_mask0:3259; CHECK: # %bb.0:3260; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [4,8,9,10]3261; CHECK-NEXT: vpermps %zmm0, %zmm1, %zmm03262; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm03263; CHECK-NEXT: vzeroupper3264; CHECK-NEXT: retq3265 %res = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 4, i32 8, i32 9, i32 10>3266 ret <4 x float> %res3267}3268define <4 x float> @test_masked_16xfloat_to_4xfloat_perm_mask0(<16 x float> %vec, <4 x float> %vec2, <4 x float> %mask) {3269; CHECK-LABEL: test_masked_16xfloat_to_4xfloat_perm_mask0:3270; CHECK: # %bb.0:3271; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm13272; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm3 = [4,8,9,10]3273; CHECK-NEXT: vxorps %xmm4, %xmm4, %xmm43274; CHECK-NEXT: vcmpeqps %xmm4, %xmm2, %k13275; CHECK-NEXT: vpermps %zmm0, %zmm3, %zmm1 {%k1}3276; CHECK-NEXT: vmovaps %xmm1, %xmm03277; CHECK-NEXT: vzeroupper3278; CHECK-NEXT: retq3279 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 4, i32 8, i32 9, i32 10>3280 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3281 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec23282 ret <4 x float> %res3283}3284 3285define <4 x float> @test_masked_z_16xfloat_to_4xfloat_perm_mask0(<16 x float> %vec, <4 x float> %mask) {3286; CHECK-LABEL: test_masked_z_16xfloat_to_4xfloat_perm_mask0:3287; CHECK: # %bb.0:3288; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [4,8,9,10]3289; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm33290; CHECK-NEXT: vcmpeqps %xmm3, %xmm1, %k13291; CHECK-NEXT: vpermps %zmm0, %zmm2, %zmm0 {%k1} {z}3292; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm03293; CHECK-NEXT: vzeroupper3294; CHECK-NEXT: retq3295 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 4, i32 8, i32 9, i32 10>3296 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3297 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer3298 ret <4 x float> %res3299}3300define <4 x float> @test_masked_16xfloat_to_4xfloat_perm_mask1(<16 x float> %vec, <4 x float> %vec2, <4 x float> %mask) {3301; CHECK-FAST-LABEL: test_masked_16xfloat_to_4xfloat_perm_mask1:3302; CHECK-FAST: # %bb.0:3303; CHECK-FAST-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm13304; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} xmm3 = [8,6,10,6]3305; CHECK-FAST-NEXT: vxorps %xmm4, %xmm4, %xmm43306; CHECK-FAST-NEXT: vcmpeqps %xmm4, %xmm2, %k13307; CHECK-FAST-NEXT: vpermps %zmm0, %zmm3, %zmm1 {%k1}3308; CHECK-FAST-NEXT: vmovaps %xmm1, %xmm03309; CHECK-FAST-NEXT: vzeroupper3310; CHECK-FAST-NEXT: retq3311;3312; CHECK-FAST-PERLANE-LABEL: test_masked_16xfloat_to_4xfloat_perm_mask1:3313; CHECK-FAST-PERLANE: # %bb.0:3314; CHECK-FAST-PERLANE-NEXT: vextractf32x4 $2, %zmm0, %xmm33315; CHECK-FAST-PERLANE-NEXT: vextractf128 $1, %ymm0, %xmm03316; CHECK-FAST-PERLANE-NEXT: vpmovsxbd {{.*#+}} xmm4 = [0,6,2,6]3317; CHECK-FAST-PERLANE-NEXT: vpermi2ps %xmm0, %xmm3, %xmm43318; CHECK-FAST-PERLANE-NEXT: vxorps %xmm0, %xmm0, %xmm03319; CHECK-FAST-PERLANE-NEXT: vcmpeqps %xmm0, %xmm2, %k13320; CHECK-FAST-PERLANE-NEXT: vblendmps %xmm4, %xmm1, %xmm0 {%k1}3321; CHECK-FAST-PERLANE-NEXT: vzeroupper3322; CHECK-FAST-PERLANE-NEXT: retq3323 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 8, i32 6, i32 10, i32 6>3324 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3325 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec23326 ret <4 x float> %res3327}3328 3329define <4 x float> @test_masked_z_16xfloat_to_4xfloat_perm_mask1(<16 x float> %vec, <4 x float> %mask) {3330; CHECK-FAST-LABEL: test_masked_z_16xfloat_to_4xfloat_perm_mask1:3331; CHECK-FAST: # %bb.0:3332; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} xmm2 = [8,6,10,6]3333; CHECK-FAST-NEXT: vxorps %xmm3, %xmm3, %xmm33334; CHECK-FAST-NEXT: vcmpeqps %xmm3, %xmm1, %k13335; CHECK-FAST-NEXT: vpermps %zmm0, %zmm2, %zmm0 {%k1} {z}3336; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm03337; CHECK-FAST-NEXT: vzeroupper3338; CHECK-FAST-NEXT: retq3339;3340; CHECK-FAST-PERLANE-LABEL: test_masked_z_16xfloat_to_4xfloat_perm_mask1:3341; CHECK-FAST-PERLANE: # %bb.0:3342; CHECK-FAST-PERLANE-NEXT: vextractf32x4 $2, %zmm0, %xmm23343; CHECK-FAST-PERLANE-NEXT: vextractf128 $1, %ymm0, %xmm33344; CHECK-FAST-PERLANE-NEXT: vpmovsxbd {{.*#+}} xmm0 = [0,6,2,6]3345; CHECK-FAST-PERLANE-NEXT: vxorps %xmm4, %xmm4, %xmm43346; CHECK-FAST-PERLANE-NEXT: vcmpeqps %xmm4, %xmm1, %k13347; CHECK-FAST-PERLANE-NEXT: vpermi2ps %xmm3, %xmm2, %xmm0 {%k1} {z}3348; CHECK-FAST-PERLANE-NEXT: vzeroupper3349; CHECK-FAST-PERLANE-NEXT: retq3350 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 8, i32 6, i32 10, i32 6>3351 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3352 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer3353 ret <4 x float> %res3354}3355define <4 x float> @test_masked_16xfloat_to_4xfloat_perm_mask2(<16 x float> %vec, <4 x float> %vec2, <4 x float> %mask) {3356; CHECK-LABEL: test_masked_16xfloat_to_4xfloat_perm_mask2:3357; CHECK: # %bb.0:3358; CHECK-NEXT: vextractf64x4 $1, %zmm0, %ymm33359; CHECK-NEXT: vshufps {{.*#+}} ymm0 = ymm3[0,0],ymm0[0,1],ymm3[4,4],ymm0[4,5]3360; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm33361; CHECK-NEXT: vcmpeqps %xmm3, %xmm2, %k13362; CHECK-NEXT: vextractf32x4 $1, %ymm0, %xmm1 {%k1}3363; CHECK-NEXT: vmovaps %xmm1, %xmm03364; CHECK-NEXT: vzeroupper3365; CHECK-NEXT: retq3366 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 12, i32 12, i32 4, i32 5>3367 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3368 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec23369 ret <4 x float> %res3370}3371 3372define <4 x float> @test_masked_z_16xfloat_to_4xfloat_perm_mask2(<16 x float> %vec, <4 x float> %mask) {3373; CHECK-LABEL: test_masked_z_16xfloat_to_4xfloat_perm_mask2:3374; CHECK: # %bb.0:3375; CHECK-NEXT: vextractf64x4 $1, %zmm0, %ymm23376; CHECK-NEXT: vshufps {{.*#+}} ymm0 = ymm2[0,0],ymm0[0,1],ymm2[4,4],ymm0[4,5]3377; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm23378; CHECK-NEXT: vcmpeqps %xmm2, %xmm1, %k13379; CHECK-NEXT: vextractf32x4 $1, %ymm0, %xmm0 {%k1} {z}3380; CHECK-NEXT: vzeroupper3381; CHECK-NEXT: retq3382 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 12, i32 12, i32 4, i32 5>3383 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3384 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer3385 ret <4 x float> %res3386}3387define <4 x float> @test_16xfloat_to_4xfloat_perm_mask3(<16 x float> %vec) {3388; CHECK-LABEL: test_16xfloat_to_4xfloat_perm_mask3:3389; CHECK: # %bb.0:3390; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [10,2,11,6]3391; CHECK-NEXT: vpermps %zmm0, %zmm1, %zmm03392; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm03393; CHECK-NEXT: vzeroupper3394; CHECK-NEXT: retq3395 %res = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 10, i32 2, i32 11, i32 6>3396 ret <4 x float> %res3397}3398define <4 x float> @test_masked_16xfloat_to_4xfloat_perm_mask3(<16 x float> %vec, <4 x float> %vec2, <4 x float> %mask) {3399; CHECK-LABEL: test_masked_16xfloat_to_4xfloat_perm_mask3:3400; CHECK: # %bb.0:3401; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm13402; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm3 = [10,2,11,6]3403; CHECK-NEXT: vxorps %xmm4, %xmm4, %xmm43404; CHECK-NEXT: vcmpeqps %xmm4, %xmm2, %k13405; CHECK-NEXT: vpermps %zmm0, %zmm3, %zmm1 {%k1}3406; CHECK-NEXT: vmovaps %xmm1, %xmm03407; CHECK-NEXT: vzeroupper3408; CHECK-NEXT: retq3409 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 10, i32 2, i32 11, i32 6>3410 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3411 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec23412 ret <4 x float> %res3413}3414 3415define <4 x float> @test_masked_z_16xfloat_to_4xfloat_perm_mask3(<16 x float> %vec, <4 x float> %mask) {3416; CHECK-LABEL: test_masked_z_16xfloat_to_4xfloat_perm_mask3:3417; CHECK: # %bb.0:3418; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [10,2,11,6]3419; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm33420; CHECK-NEXT: vcmpeqps %xmm3, %xmm1, %k13421; CHECK-NEXT: vpermps %zmm0, %zmm2, %zmm0 {%k1} {z}3422; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm03423; CHECK-NEXT: vzeroupper3424; CHECK-NEXT: retq3425 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 10, i32 2, i32 11, i32 6>3426 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3427 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer3428 ret <4 x float> %res3429}3430define <8 x float> @test_16xfloat_to_8xfloat_perm_mem_mask0(ptr %vp) {3431; CHECK-LABEL: test_16xfloat_to_8xfloat_perm_mem_mask0:3432; CHECK: # %bb.0:3433; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm0 = [7,6,7,11,5,10,0,4]3434; CHECK-NEXT: vpermps (%rdi), %zmm0, %zmm03435; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03436; CHECK-NEXT: retq3437 %vec = load <16 x float>, ptr %vp3438 %res = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 7, i32 6, i32 7, i32 11, i32 5, i32 10, i32 0, i32 4>3439 ret <8 x float> %res3440}3441define <8 x float> @test_masked_16xfloat_to_8xfloat_perm_mem_mask0(ptr %vp, <8 x float> %vec2, <8 x float> %mask) {3442; CHECK-LABEL: test_masked_16xfloat_to_8xfloat_perm_mem_mask0:3443; CHECK: # %bb.0:3444; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm03445; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [7,6,7,11,5,10,0,4]3446; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm33447; CHECK-NEXT: vcmpeqps %ymm3, %ymm1, %k13448; CHECK-NEXT: vpermps (%rdi), %zmm2, %zmm0 {%k1}3449; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03450; CHECK-NEXT: retq3451 %vec = load <16 x float>, ptr %vp3452 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 7, i32 6, i32 7, i32 11, i32 5, i32 10, i32 0, i32 4>3453 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer3454 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec23455 ret <8 x float> %res3456}3457 3458define <8 x float> @test_masked_z_16xfloat_to_8xfloat_perm_mem_mask0(ptr %vp, <8 x float> %mask) {3459; CHECK-LABEL: test_masked_z_16xfloat_to_8xfloat_perm_mem_mask0:3460; CHECK: # %bb.0:3461; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [7,6,7,11,5,10,0,4]3462; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm23463; CHECK-NEXT: vcmpeqps %ymm2, %ymm0, %k13464; CHECK-NEXT: vpermps (%rdi), %zmm1, %zmm0 {%k1} {z}3465; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03466; CHECK-NEXT: retq3467 %vec = load <16 x float>, ptr %vp3468 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 7, i32 6, i32 7, i32 11, i32 5, i32 10, i32 0, i32 4>3469 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer3470 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer3471 ret <8 x float> %res3472}3473 3474define <8 x float> @test_masked_16xfloat_to_8xfloat_perm_mem_mask1(ptr %vp, <8 x float> %vec2, <8 x float> %mask) {3475; CHECK-LABEL: test_masked_16xfloat_to_8xfloat_perm_mem_mask1:3476; CHECK: # %bb.0:3477; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm03478; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [11,0,9,0,7,14,0,8]3479; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm33480; CHECK-NEXT: vcmpeqps %ymm3, %ymm1, %k13481; CHECK-NEXT: vpermps (%rdi), %zmm2, %zmm0 {%k1}3482; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03483; CHECK-NEXT: retq3484 %vec = load <16 x float>, ptr %vp3485 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 11, i32 0, i32 9, i32 0, i32 7, i32 14, i32 0, i32 8>3486 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer3487 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec23488 ret <8 x float> %res3489}3490 3491define <8 x float> @test_masked_z_16xfloat_to_8xfloat_perm_mem_mask1(ptr %vp, <8 x float> %mask) {3492; CHECK-LABEL: test_masked_z_16xfloat_to_8xfloat_perm_mem_mask1:3493; CHECK: # %bb.0:3494; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [11,0,9,0,7,14,0,8]3495; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm23496; CHECK-NEXT: vcmpeqps %ymm2, %ymm0, %k13497; CHECK-NEXT: vpermps (%rdi), %zmm1, %zmm0 {%k1} {z}3498; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03499; CHECK-NEXT: retq3500 %vec = load <16 x float>, ptr %vp3501 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 11, i32 0, i32 9, i32 0, i32 7, i32 14, i32 0, i32 8>3502 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer3503 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer3504 ret <8 x float> %res3505}3506 3507define <8 x float> @test_masked_16xfloat_to_8xfloat_perm_mem_mask2(ptr %vp, <8 x float> %vec2, <8 x float> %mask) {3508; CHECK-FAST-LABEL: test_masked_16xfloat_to_8xfloat_perm_mem_mask2:3509; CHECK-FAST: # %bb.0:3510; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm03511; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm2 = [1,13,10,11,10,0,0,9]3512; CHECK-FAST-NEXT: vxorps %xmm3, %xmm3, %xmm33513; CHECK-FAST-NEXT: vcmpeqps %ymm3, %ymm1, %k13514; CHECK-FAST-NEXT: vpermps (%rdi), %zmm2, %zmm0 {%k1}3515; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03516; CHECK-FAST-NEXT: retq3517;3518; CHECK-FAST-PERLANE-LABEL: test_masked_16xfloat_to_8xfloat_perm_mem_mask2:3519; CHECK-FAST-PERLANE: # %bb.0:3520; CHECK-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm23521; CHECK-FAST-PERLANE-NEXT: vpmovsxbd {{.*#+}} ymm3 = [1,13,10,11,10,0,0,9]3522; CHECK-FAST-PERLANE-NEXT: vpermi2ps 32(%rdi), %ymm2, %ymm33523; CHECK-FAST-PERLANE-NEXT: vxorps %xmm2, %xmm2, %xmm23524; CHECK-FAST-PERLANE-NEXT: vcmpeqps %ymm2, %ymm1, %k13525; CHECK-FAST-PERLANE-NEXT: vmovaps %ymm3, %ymm0 {%k1}3526; CHECK-FAST-PERLANE-NEXT: retq3527 %vec = load <16 x float>, ptr %vp3528 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 1, i32 13, i32 10, i32 11, i32 10, i32 0, i32 0, i32 9>3529 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer3530 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec23531 ret <8 x float> %res3532}3533 3534define <8 x float> @test_masked_z_16xfloat_to_8xfloat_perm_mem_mask2(ptr %vp, <8 x float> %mask) {3535; CHECK-FAST-LABEL: test_masked_z_16xfloat_to_8xfloat_perm_mem_mask2:3536; CHECK-FAST: # %bb.0:3537; CHECK-FAST-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,13,10,11,10,0,0,9]3538; CHECK-FAST-NEXT: vxorps %xmm2, %xmm2, %xmm23539; CHECK-FAST-NEXT: vcmpeqps %ymm2, %ymm0, %k13540; CHECK-FAST-NEXT: vpermps (%rdi), %zmm1, %zmm0 {%k1} {z}3541; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03542; CHECK-FAST-NEXT: retq3543;3544; CHECK-FAST-PERLANE-LABEL: test_masked_z_16xfloat_to_8xfloat_perm_mem_mask2:3545; CHECK-FAST-PERLANE: # %bb.0:3546; CHECK-FAST-PERLANE-NEXT: vmovaps (%rdi), %xmm23547; CHECK-FAST-PERLANE-NEXT: vpmovsxbd {{.*#+}} ymm1 = [1,13,10,11,10,0,0,9]3548; CHECK-FAST-PERLANE-NEXT: vxorps %xmm3, %xmm3, %xmm33549; CHECK-FAST-PERLANE-NEXT: vcmpeqps %ymm3, %ymm0, %k13550; CHECK-FAST-PERLANE-NEXT: vpermi2ps 32(%rdi), %ymm2, %ymm1 {%k1} {z}3551; CHECK-FAST-PERLANE-NEXT: vmovaps %ymm1, %ymm03552; CHECK-FAST-PERLANE-NEXT: retq3553 %vec = load <16 x float>, ptr %vp3554 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 1, i32 13, i32 10, i32 11, i32 10, i32 0, i32 0, i32 9>3555 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer3556 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer3557 ret <8 x float> %res3558}3559 3560define <8 x float> @test_16xfloat_to_8xfloat_perm_mem_mask3(ptr %vp) {3561; CHECK-LABEL: test_16xfloat_to_8xfloat_perm_mem_mask3:3562; CHECK: # %bb.0:3563; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm0 = [15,13,11,11,3,12,4,1]3564; CHECK-NEXT: vpermps (%rdi), %zmm0, %zmm03565; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03566; CHECK-NEXT: retq3567 %vec = load <16 x float>, ptr %vp3568 %res = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 15, i32 13, i32 11, i32 11, i32 3, i32 12, i32 4, i32 1>3569 ret <8 x float> %res3570}3571define <8 x float> @test_masked_16xfloat_to_8xfloat_perm_mem_mask3(ptr %vp, <8 x float> %vec2, <8 x float> %mask) {3572; CHECK-LABEL: test_masked_16xfloat_to_8xfloat_perm_mem_mask3:3573; CHECK: # %bb.0:3574; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm03575; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [15,13,11,11,3,12,4,1]3576; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm33577; CHECK-NEXT: vcmpeqps %ymm3, %ymm1, %k13578; CHECK-NEXT: vpermps (%rdi), %zmm2, %zmm0 {%k1}3579; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03580; CHECK-NEXT: retq3581 %vec = load <16 x float>, ptr %vp3582 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 15, i32 13, i32 11, i32 11, i32 3, i32 12, i32 4, i32 1>3583 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer3584 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec23585 ret <8 x float> %res3586}3587 3588define <8 x float> @test_masked_z_16xfloat_to_8xfloat_perm_mem_mask3(ptr %vp, <8 x float> %mask) {3589; CHECK-LABEL: test_masked_z_16xfloat_to_8xfloat_perm_mem_mask3:3590; CHECK: # %bb.0:3591; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [15,13,11,11,3,12,4,1]3592; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm23593; CHECK-NEXT: vcmpeqps %ymm2, %ymm0, %k13594; CHECK-NEXT: vpermps (%rdi), %zmm1, %zmm0 {%k1} {z}3595; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03596; CHECK-NEXT: retq3597 %vec = load <16 x float>, ptr %vp3598 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <8 x i32> <i32 15, i32 13, i32 11, i32 11, i32 3, i32 12, i32 4, i32 1>3599 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer3600 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer3601 ret <8 x float> %res3602}3603 3604define <4 x float> @test_16xfloat_to_4xfloat_perm_mem_mask0(ptr %vp) {3605; CHECK-LABEL: test_16xfloat_to_4xfloat_perm_mem_mask0:3606; CHECK: # %bb.0:3607; CHECK-NEXT: vpermpd $231, 32(%rdi), %ymm1 # ymm1 = mem[3,1,2,3]3608; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm0 = [0,6,7,3]3609; CHECK-NEXT: vpermi2ps 16(%rdi), %xmm1, %xmm03610; CHECK-NEXT: vzeroupper3611; CHECK-NEXT: retq3612 %vec = load <16 x float>, ptr %vp3613 %res = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 14, i32 6, i32 7, i32 11>3614 ret <4 x float> %res3615}3616define <4 x float> @test_masked_16xfloat_to_4xfloat_perm_mem_mask0(ptr %vp, <4 x float> %vec2, <4 x float> %mask) {3617; CHECK-LABEL: test_masked_16xfloat_to_4xfloat_perm_mem_mask0:3618; CHECK: # %bb.0:3619; CHECK-NEXT: vpermpd $231, 32(%rdi), %ymm2 # ymm2 = mem[3,1,2,3]3620; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm3 = [0,6,7,3]3621; CHECK-NEXT: vpermi2ps 16(%rdi), %xmm2, %xmm33622; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm23623; CHECK-NEXT: vcmpeqps %xmm2, %xmm1, %k13624; CHECK-NEXT: vmovaps %xmm3, %xmm0 {%k1}3625; CHECK-NEXT: vzeroupper3626; CHECK-NEXT: retq3627 %vec = load <16 x float>, ptr %vp3628 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 14, i32 6, i32 7, i32 11>3629 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3630 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec23631 ret <4 x float> %res3632}3633 3634define <4 x float> @test_masked_z_16xfloat_to_4xfloat_perm_mem_mask0(ptr %vp, <4 x float> %mask) {3635; CHECK-LABEL: test_masked_z_16xfloat_to_4xfloat_perm_mem_mask0:3636; CHECK: # %bb.0:3637; CHECK-NEXT: vpermpd $231, 32(%rdi), %ymm2 # ymm2 = mem[3,1,2,3]3638; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [0,6,7,3]3639; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm33640; CHECK-NEXT: vcmpeqps %xmm3, %xmm0, %k13641; CHECK-NEXT: vpermi2ps 16(%rdi), %xmm2, %xmm1 {%k1} {z}3642; CHECK-NEXT: vmovaps %xmm1, %xmm03643; CHECK-NEXT: vzeroupper3644; CHECK-NEXT: retq3645 %vec = load <16 x float>, ptr %vp3646 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 14, i32 6, i32 7, i32 11>3647 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3648 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer3649 ret <4 x float> %res3650}3651 3652define <4 x float> @test_masked_16xfloat_to_4xfloat_perm_mem_mask1(ptr %vp, <4 x float> %vec2, <4 x float> %mask) {3653; CHECK-LABEL: test_masked_16xfloat_to_4xfloat_perm_mem_mask1:3654; CHECK: # %bb.0:3655; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm03656; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm2 = [8,2,14,7,12,6,14,7]3657; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm33658; CHECK-NEXT: vcmpeqps %xmm3, %xmm1, %k13659; CHECK-NEXT: vpermps (%rdi), %zmm2, %zmm0 {%k1}3660; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm03661; CHECK-NEXT: vzeroupper3662; CHECK-NEXT: retq3663 %vec = load <16 x float>, ptr %vp3664 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 8, i32 2, i32 14, i32 7>3665 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3666 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec23667 ret <4 x float> %res3668}3669 3670define <4 x float> @test_masked_z_16xfloat_to_4xfloat_perm_mem_mask1(ptr %vp, <4 x float> %mask) {3671; CHECK-LABEL: test_masked_z_16xfloat_to_4xfloat_perm_mem_mask1:3672; CHECK: # %bb.0:3673; CHECK-NEXT: vpmovsxbd {{.*#+}} ymm1 = [8,2,14,7,12,6,14,7]3674; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm23675; CHECK-NEXT: vcmpeqps %xmm2, %xmm0, %k13676; CHECK-NEXT: vpermps (%rdi), %zmm1, %zmm0 {%k1} {z}3677; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm03678; CHECK-NEXT: vzeroupper3679; CHECK-NEXT: retq3680 %vec = load <16 x float>, ptr %vp3681 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 8, i32 2, i32 14, i32 7>3682 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3683 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer3684 ret <4 x float> %res3685}3686 3687define <4 x float> @test_masked_16xfloat_to_4xfloat_perm_mem_mask2(ptr %vp, <4 x float> %vec2, <4 x float> %mask) {3688; CHECK-LABEL: test_masked_16xfloat_to_4xfloat_perm_mem_mask2:3689; CHECK: # %bb.0:3690; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm03691; CHECK-NEXT: vbroadcastsd {{.*#+}} ymm2 = [12,6,12,6,12,6,12,6]3692; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm33693; CHECK-NEXT: vcmpeqps %xmm3, %xmm1, %k13694; CHECK-NEXT: vpermps (%rdi), %zmm2, %zmm0 {%k1}3695; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm03696; CHECK-NEXT: vzeroupper3697; CHECK-NEXT: retq3698 %vec = load <16 x float>, ptr %vp3699 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 12, i32 6, i32 12, i32 6>3700 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3701 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec23702 ret <4 x float> %res3703}3704 3705define <4 x float> @test_masked_z_16xfloat_to_4xfloat_perm_mem_mask2(ptr %vp, <4 x float> %mask) {3706; CHECK-LABEL: test_masked_z_16xfloat_to_4xfloat_perm_mem_mask2:3707; CHECK: # %bb.0:3708; CHECK-NEXT: vbroadcastsd {{.*#+}} ymm1 = [12,6,12,6,12,6,12,6]3709; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm23710; CHECK-NEXT: vcmpeqps %xmm2, %xmm0, %k13711; CHECK-NEXT: vpermps (%rdi), %zmm1, %zmm0 {%k1} {z}3712; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm03713; CHECK-NEXT: vzeroupper3714; CHECK-NEXT: retq3715 %vec = load <16 x float>, ptr %vp3716 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 12, i32 6, i32 12, i32 6>3717 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3718 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer3719 ret <4 x float> %res3720}3721 3722define <4 x float> @test_16xfloat_to_4xfloat_perm_mem_mask3(ptr %vp) {3723; CHECK-LABEL: test_16xfloat_to_4xfloat_perm_mem_mask3:3724; CHECK: # %bb.0:3725; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm0 = [3,3,15,9]3726; CHECK-NEXT: vpermps (%rdi), %zmm0, %zmm03727; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm03728; CHECK-NEXT: vzeroupper3729; CHECK-NEXT: retq3730 %vec = load <16 x float>, ptr %vp3731 %res = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 3, i32 3, i32 15, i32 9>3732 ret <4 x float> %res3733}3734define <4 x float> @test_masked_16xfloat_to_4xfloat_perm_mem_mask3(ptr %vp, <4 x float> %vec2, <4 x float> %mask) {3735; CHECK-LABEL: test_masked_16xfloat_to_4xfloat_perm_mem_mask3:3736; CHECK: # %bb.0:3737; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm03738; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm2 = [3,3,15,9]3739; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm33740; CHECK-NEXT: vcmpeqps %xmm3, %xmm1, %k13741; CHECK-NEXT: vpermps (%rdi), %zmm2, %zmm0 {%k1}3742; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm03743; CHECK-NEXT: vzeroupper3744; CHECK-NEXT: retq3745 %vec = load <16 x float>, ptr %vp3746 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 3, i32 3, i32 15, i32 9>3747 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3748 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec23749 ret <4 x float> %res3750}3751 3752define <4 x float> @test_masked_z_16xfloat_to_4xfloat_perm_mem_mask3(ptr %vp, <4 x float> %mask) {3753; CHECK-LABEL: test_masked_z_16xfloat_to_4xfloat_perm_mem_mask3:3754; CHECK: # %bb.0:3755; CHECK-NEXT: vpmovsxbd {{.*#+}} xmm1 = [3,3,15,9]3756; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm23757; CHECK-NEXT: vcmpeqps %xmm2, %xmm0, %k13758; CHECK-NEXT: vpermps (%rdi), %zmm1, %zmm0 {%k1} {z}3759; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm03760; CHECK-NEXT: vzeroupper3761; CHECK-NEXT: retq3762 %vec = load <16 x float>, ptr %vp3763 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <4 x i32> <i32 3, i32 3, i32 15, i32 9>3764 %cmp = fcmp oeq <4 x float> %mask, zeroinitializer3765 %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer3766 ret <4 x float> %res3767}3768 3769define <2 x double> @test_4xdouble_to_2xdouble_perm_mask0(<4 x double> %vec) {3770; CHECK-LABEL: test_4xdouble_to_2xdouble_perm_mask0:3771; CHECK: # %bb.0:3772; CHECK-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[2,0,2,3]3773; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm03774; CHECK-NEXT: vzeroupper3775; CHECK-NEXT: retq3776 %res = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 0>3777 ret <2 x double> %res3778}3779define <2 x double> @test_masked_4xdouble_to_2xdouble_perm_mask0(<4 x double> %vec, <2 x double> %vec2, <2 x double> %mask) {3780; CHECK-LABEL: test_masked_4xdouble_to_2xdouble_perm_mask0:3781; CHECK: # %bb.0:3782; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm13783; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm33784; CHECK-NEXT: vcmpeqpd %xmm3, %xmm2, %k13785; CHECK-NEXT: vpermpd {{.*#+}} ymm1 {%k1} = ymm0[2,0,2,3]3786; CHECK-NEXT: vmovapd %xmm1, %xmm03787; CHECK-NEXT: vzeroupper3788; CHECK-NEXT: retq3789 %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 0>3790 %cmp = fcmp oeq <2 x double> %mask, zeroinitializer3791 %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> %vec23792 ret <2 x double> %res3793}3794 3795define <2 x double> @test_masked_z_4xdouble_to_2xdouble_perm_mask0(<4 x double> %vec, <2 x double> %mask) {3796; CHECK-LABEL: test_masked_z_4xdouble_to_2xdouble_perm_mask0:3797; CHECK: # %bb.0:3798; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm23799; CHECK-NEXT: vcmpeqpd %xmm2, %xmm1, %k13800; CHECK-NEXT: vpermpd {{.*#+}} ymm0 {%k1} {z} = ymm0[2,0,2,3]3801; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm03802; CHECK-NEXT: vzeroupper3803; CHECK-NEXT: retq3804 %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 0>3805 %cmp = fcmp oeq <2 x double> %mask, zeroinitializer3806 %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> zeroinitializer3807 ret <2 x double> %res3808}3809define <2 x double> @test_masked_4xdouble_to_2xdouble_perm_mask1(<4 x double> %vec, <2 x double> %vec2, <2 x double> %mask) {3810; CHECK-LABEL: test_masked_4xdouble_to_2xdouble_perm_mask1:3811; CHECK: # %bb.0:3812; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $ymm13813; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm33814; CHECK-NEXT: vcmpeqpd %xmm3, %xmm2, %k13815; CHECK-NEXT: vpermpd {{.*#+}} ymm1 {%k1} = ymm0[1,3,2,3]3816; CHECK-NEXT: vmovapd %xmm1, %xmm03817; CHECK-NEXT: vzeroupper3818; CHECK-NEXT: retq3819 %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 1, i32 3>3820 %cmp = fcmp oeq <2 x double> %mask, zeroinitializer3821 %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> %vec23822 ret <2 x double> %res3823}3824 3825define <2 x double> @test_masked_z_4xdouble_to_2xdouble_perm_mask1(<4 x double> %vec, <2 x double> %mask) {3826; CHECK-LABEL: test_masked_z_4xdouble_to_2xdouble_perm_mask1:3827; CHECK: # %bb.0:3828; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm23829; CHECK-NEXT: vcmpeqpd %xmm2, %xmm1, %k13830; CHECK-NEXT: vpermpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1,3,2,3]3831; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm03832; CHECK-NEXT: vzeroupper3833; CHECK-NEXT: retq3834 %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 1, i32 3>3835 %cmp = fcmp oeq <2 x double> %mask, zeroinitializer3836 %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> zeroinitializer3837 ret <2 x double> %res3838}3839define <2 x double> @test_4xdouble_to_2xdouble_perm_mem_mask0(ptr %vp) {3840; CHECK-LABEL: test_4xdouble_to_2xdouble_perm_mem_mask0:3841; CHECK: # %bb.0:3842; CHECK-NEXT: vmovaps (%rdi), %xmm03843; CHECK-NEXT: vblendps $3, 16(%rdi), %xmm0, %xmm0 # xmm0 = mem[0,1],xmm0[2,3]3844; CHECK-NEXT: retq3845 %vec = load <4 x double>, ptr %vp3846 %res = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 1>3847 ret <2 x double> %res3848}3849define <2 x double> @test_masked_4xdouble_to_2xdouble_perm_mem_mask0(ptr %vp, <2 x double> %vec2, <2 x double> %mask) {3850; CHECK-LABEL: test_masked_4xdouble_to_2xdouble_perm_mem_mask0:3851; CHECK: # %bb.0:3852; CHECK-NEXT: vmovapd (%rdi), %xmm23853; CHECK-NEXT: vblendpd $1, 16(%rdi), %xmm2, %xmm2 # xmm2 = mem[0],xmm2[1]3854; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm33855; CHECK-NEXT: vcmpeqpd %xmm3, %xmm1, %k13856; CHECK-NEXT: vmovapd %xmm2, %xmm0 {%k1}3857; CHECK-NEXT: retq3858 %vec = load <4 x double>, ptr %vp3859 %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 1>3860 %cmp = fcmp oeq <2 x double> %mask, zeroinitializer3861 %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> %vec23862 ret <2 x double> %res3863}3864 3865define <2 x double> @test_masked_z_4xdouble_to_2xdouble_perm_mem_mask0(ptr %vp, <2 x double> %mask) {3866; CHECK-LABEL: test_masked_z_4xdouble_to_2xdouble_perm_mem_mask0:3867; CHECK: # %bb.0:3868; CHECK-NEXT: vmovapd (%rdi), %xmm13869; CHECK-NEXT: vblendpd $1, 16(%rdi), %xmm1, %xmm1 # xmm1 = mem[0],xmm1[1]3870; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm23871; CHECK-NEXT: vcmpeqpd %xmm2, %xmm0, %k13872; CHECK-NEXT: vmovapd %xmm1, %xmm0 {%k1} {z}3873; CHECK-NEXT: retq3874 %vec = load <4 x double>, ptr %vp3875 %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 1>3876 %cmp = fcmp oeq <2 x double> %mask, zeroinitializer3877 %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> zeroinitializer3878 ret <2 x double> %res3879}3880 3881define <2 x double> @test_masked_4xdouble_to_2xdouble_perm_mem_mask1(ptr %vp, <2 x double> %vec2, <2 x double> %mask) {3882; CHECK-LABEL: test_masked_4xdouble_to_2xdouble_perm_mem_mask1:3883; CHECK: # %bb.0:3884; CHECK-NEXT: vmovapd 16(%rdi), %xmm23885; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm33886; CHECK-NEXT: vcmpeqpd %xmm3, %xmm1, %k13887; CHECK-NEXT: vunpcklpd (%rdi), %xmm2, %xmm0 {%k1} # xmm0 {%k1} = xmm2[0],mem[0]3888; CHECK-NEXT: retq3889 %vec = load <4 x double>, ptr %vp3890 %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 0>3891 %cmp = fcmp oeq <2 x double> %mask, zeroinitializer3892 %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> %vec23893 ret <2 x double> %res3894}3895 3896define <2 x double> @test_masked_z_4xdouble_to_2xdouble_perm_mem_mask1(ptr %vp, <2 x double> %mask) {3897; CHECK-LABEL: test_masked_z_4xdouble_to_2xdouble_perm_mem_mask1:3898; CHECK: # %bb.0:3899; CHECK-NEXT: vmovapd 16(%rdi), %xmm13900; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm23901; CHECK-NEXT: vcmpeqpd %xmm2, %xmm0, %k13902; CHECK-NEXT: vunpcklpd (%rdi), %xmm1, %xmm0 {%k1} {z} # xmm0 {%k1} {z} = xmm1[0],mem[0]3903; CHECK-NEXT: retq3904 %vec = load <4 x double>, ptr %vp3905 %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 0>3906 %cmp = fcmp oeq <2 x double> %mask, zeroinitializer3907 %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> zeroinitializer3908 ret <2 x double> %res3909}3910 3911define <4 x double> @test_8xdouble_to_4xdouble_perm_mask0(<8 x double> %vec) {3912; CHECK-LABEL: test_8xdouble_to_4xdouble_perm_mask0:3913; CHECK: # %bb.0:3914; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,3,7,3]3915; CHECK-NEXT: vpermpd %zmm0, %zmm1, %zmm03916; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03917; CHECK-NEXT: retq3918 %res = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 7, i32 3, i32 7, i32 3>3919 ret <4 x double> %res3920}3921define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask0(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {3922; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask0:3923; CHECK: # %bb.0:3924; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm13925; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [7,3,7,3]3926; CHECK-NEXT: vxorpd %xmm4, %xmm4, %xmm43927; CHECK-NEXT: vcmpeqpd %ymm4, %ymm2, %k13928; CHECK-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}3929; CHECK-NEXT: vmovapd %ymm1, %ymm03930; CHECK-NEXT: retq3931 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 7, i32 3, i32 7, i32 3>3932 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer3933 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec23934 ret <4 x double> %res3935}3936 3937define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask0(<8 x double> %vec, <4 x double> %mask) {3938; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask0:3939; CHECK: # %bb.0:3940; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [7,3,7,3]3941; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm33942; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k13943; CHECK-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}3944; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03945; CHECK-NEXT: retq3946 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 7, i32 3, i32 7, i32 3>3947 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer3948 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer3949 ret <4 x double> %res3950}3951define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask1(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {3952; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask1:3953; CHECK: # %bb.0:3954; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm13955; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [2,0,7,6]3956; CHECK-NEXT: vxorpd %xmm4, %xmm4, %xmm43957; CHECK-NEXT: vcmpeqpd %ymm4, %ymm2, %k13958; CHECK-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}3959; CHECK-NEXT: vmovapd %ymm1, %ymm03960; CHECK-NEXT: retq3961 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 2, i32 0, i32 7, i32 6>3962 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer3963 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec23964 ret <4 x double> %res3965}3966 3967define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask1(<8 x double> %vec, <4 x double> %mask) {3968; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask1:3969; CHECK: # %bb.0:3970; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [2,0,7,6]3971; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm33972; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k13973; CHECK-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}3974; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm03975; CHECK-NEXT: retq3976 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 2, i32 0, i32 7, i32 6>3977 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer3978 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer3979 ret <4 x double> %res3980}3981define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask2(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {3982; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask2:3983; CHECK: # %bb.0:3984; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm33985; CHECK-NEXT: vcmpeqpd %ymm3, %ymm2, %k13986; CHECK-NEXT: vpermpd {{.*#+}} ymm1 {%k1} = ymm0[2,3,2,0]3987; CHECK-NEXT: vmovapd %ymm1, %ymm03988; CHECK-NEXT: retq3989 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 2, i32 0>3990 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer3991 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec23992 ret <4 x double> %res3993}3994 3995define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask2(<8 x double> %vec, <4 x double> %mask) {3996; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask2:3997; CHECK: # %bb.0:3998; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm23999; CHECK-NEXT: vcmpeqpd %ymm2, %ymm1, %k14000; CHECK-NEXT: vpermpd {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3,2,0]4001; CHECK-NEXT: retq4002 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 2, i32 3, i32 2, i32 0>4003 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4004 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer4005 ret <4 x double> %res4006}4007define <4 x double> @test_8xdouble_to_4xdouble_perm_mask3(<8 x double> %vec) {4008; CHECK-LABEL: test_8xdouble_to_4xdouble_perm_mask3:4009; CHECK: # %bb.0:4010; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [0,2,1,4]4011; CHECK-NEXT: vpermpd %zmm0, %zmm1, %zmm04012; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04013; CHECK-NEXT: retq4014 %res = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 4>4015 ret <4 x double> %res4016}4017define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask3(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {4018; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask3:4019; CHECK: # %bb.0:4020; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm14021; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [0,2,1,4]4022; CHECK-NEXT: vxorpd %xmm4, %xmm4, %xmm44023; CHECK-NEXT: vcmpeqpd %ymm4, %ymm2, %k14024; CHECK-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}4025; CHECK-NEXT: vmovapd %ymm1, %ymm04026; CHECK-NEXT: retq4027 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 4>4028 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4029 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec24030 ret <4 x double> %res4031}4032 4033define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask3(<8 x double> %vec, <4 x double> %mask) {4034; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask3:4035; CHECK: # %bb.0:4036; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [0,2,1,4]4037; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm34038; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k14039; CHECK-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}4040; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04041; CHECK-NEXT: retq4042 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 4>4043 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4044 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer4045 ret <4 x double> %res4046}4047define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask4(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {4048; CHECK-FAST-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask4:4049; CHECK-FAST: # %bb.0:4050; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} xmm3 = [1,5]4051; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm3, %zmm04052; CHECK-FAST-NEXT: vxorpd %xmm3, %xmm3, %xmm34053; CHECK-FAST-NEXT: vcmpeqpd %ymm3, %ymm2, %k14054; CHECK-FAST-NEXT: vpermpd {{.*#+}} ymm1 {%k1} = ymm0[0,0,1,1]4055; CHECK-FAST-NEXT: vmovapd %ymm1, %ymm04056; CHECK-FAST-NEXT: retq4057;4058; CHECK-FAST-PERLANE-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask4:4059; CHECK-FAST-PERLANE: # %bb.0:4060; CHECK-FAST-PERLANE-NEXT: vextractf32x4 $2, %zmm0, %xmm34061; CHECK-FAST-PERLANE-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm3[1]4062; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm3, %xmm3, %xmm34063; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm3, %ymm2, %k14064; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm1 {%k1} = ymm0[0,0,1,1]4065; CHECK-FAST-PERLANE-NEXT: vmovapd %ymm1, %ymm04066; CHECK-FAST-PERLANE-NEXT: retq4067 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 1, i32 1, i32 5, i32 5>4068 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4069 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec24070 ret <4 x double> %res4071}4072 4073define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask4(<8 x double> %vec, <4 x double> %mask) {4074; CHECK-FAST-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask4:4075; CHECK-FAST: # %bb.0:4076; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} xmm2 = [1,5]4077; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm2, %zmm04078; CHECK-FAST-NEXT: vxorpd %xmm2, %xmm2, %xmm24079; CHECK-FAST-NEXT: vcmpeqpd %ymm2, %ymm1, %k14080; CHECK-FAST-NEXT: vpermpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0,0,1,1]4081; CHECK-FAST-NEXT: retq4082;4083; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask4:4084; CHECK-FAST-PERLANE: # %bb.0:4085; CHECK-FAST-PERLANE-NEXT: vextractf32x4 $2, %zmm0, %xmm24086; CHECK-FAST-PERLANE-NEXT: vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm2[1]4087; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm2, %xmm2, %xmm24088; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm2, %ymm1, %k14089; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0,0,1,1]4090; CHECK-FAST-PERLANE-NEXT: retq4091 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 1, i32 1, i32 5, i32 5>4092 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4093 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer4094 ret <4 x double> %res4095}4096define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask5(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {4097; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask5:4098; CHECK: # %bb.0:4099; CHECK-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm14100; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm3 = [2,6,2,2]4101; CHECK-NEXT: vxorpd %xmm4, %xmm4, %xmm44102; CHECK-NEXT: vcmpeqpd %ymm4, %ymm2, %k14103; CHECK-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}4104; CHECK-NEXT: vmovapd %ymm1, %ymm04105; CHECK-NEXT: retq4106 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 2, i32 6, i32 2, i32 2>4107 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4108 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec24109 ret <4 x double> %res4110}4111 4112define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask5(<8 x double> %vec, <4 x double> %mask) {4113; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask5:4114; CHECK: # %bb.0:4115; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [2,6,2,2]4116; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm34117; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k14118; CHECK-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}4119; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04120; CHECK-NEXT: retq4121 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 2, i32 6, i32 2, i32 2>4122 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4123 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer4124 ret <4 x double> %res4125}4126define <4 x double> @test_8xdouble_to_4xdouble_perm_mask6(<8 x double> %vec) {4127; CHECK-FAST-LABEL: test_8xdouble_to_4xdouble_perm_mask6:4128; CHECK-FAST: # %bb.0:4129; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [5,0,7,0]4130; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm1, %zmm04131; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04132; CHECK-FAST-NEXT: retq4133;4134; CHECK-FAST-PERLANE-LABEL: test_8xdouble_to_4xdouble_perm_mask6:4135; CHECK-FAST-PERLANE: # %bb.0:4136; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm14137; CHECK-FAST-PERLANE-NEXT: vbroadcastsd %xmm0, %ymm04138; CHECK-FAST-PERLANE-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]4139; CHECK-FAST-PERLANE-NEXT: retq4140 %res = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 5, i32 0, i32 7, i32 0>4141 ret <4 x double> %res4142}4143define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask6(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {4144; CHECK-FAST-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask6:4145; CHECK-FAST: # %bb.0:4146; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm14147; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [5,0,7,0]4148; CHECK-FAST-NEXT: vxorpd %xmm4, %xmm4, %xmm44149; CHECK-FAST-NEXT: vcmpeqpd %ymm4, %ymm2, %k14150; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}4151; CHECK-FAST-NEXT: vmovapd %ymm1, %ymm04152; CHECK-FAST-NEXT: retq4153;4154; CHECK-FAST-PERLANE-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask6:4155; CHECK-FAST-PERLANE: # %bb.0:4156; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm34157; CHECK-FAST-PERLANE-NEXT: vbroadcastsd %xmm0, %ymm04158; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm4, %xmm4, %xmm44159; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm4, %ymm2, %k14160; CHECK-FAST-PERLANE-NEXT: vunpckhpd {{.*#+}} ymm1 {%k1} = ymm3[1],ymm0[1],ymm3[3],ymm0[3]4161; CHECK-FAST-PERLANE-NEXT: vmovapd %ymm1, %ymm04162; CHECK-FAST-PERLANE-NEXT: retq4163 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 5, i32 0, i32 7, i32 0>4164 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4165 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec24166 ret <4 x double> %res4167}4168 4169define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask6(<8 x double> %vec, <4 x double> %mask) {4170; CHECK-FAST-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask6:4171; CHECK-FAST: # %bb.0:4172; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [5,0,7,0]4173; CHECK-FAST-NEXT: vxorpd %xmm3, %xmm3, %xmm34174; CHECK-FAST-NEXT: vcmpeqpd %ymm3, %ymm1, %k14175; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}4176; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04177; CHECK-FAST-NEXT: retq4178;4179; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask6:4180; CHECK-FAST-PERLANE: # %bb.0:4181; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm24182; CHECK-FAST-PERLANE-NEXT: vbroadcastsd %xmm0, %ymm04183; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm3, %xmm3, %xmm34184; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm3, %ymm1, %k14185; CHECK-FAST-PERLANE-NEXT: vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm2[1],ymm0[1],ymm2[3],ymm0[3]4186; CHECK-FAST-PERLANE-NEXT: retq4187 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 5, i32 0, i32 7, i32 0>4188 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4189 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer4190 ret <4 x double> %res4191}4192define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask7(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {4193; CHECK-FAST-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask7:4194; CHECK-FAST: # %bb.0:4195; CHECK-FAST-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm14196; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [3,5,0,6]4197; CHECK-FAST-NEXT: vxorpd %xmm4, %xmm4, %xmm44198; CHECK-FAST-NEXT: vcmpeqpd %ymm4, %ymm2, %k14199; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}4200; CHECK-FAST-NEXT: vmovapd %ymm1, %ymm04201; CHECK-FAST-NEXT: retq4202;4203; CHECK-FAST-PERLANE-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask7:4204; CHECK-FAST-PERLANE: # %bb.0:4205; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm34206; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,3,0,3]4207; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm4, %xmm4, %xmm44208; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm4, %ymm2, %k14209; CHECK-FAST-PERLANE-NEXT: vshufpd {{.*#+}} ymm1 {%k1} = ymm0[1],ymm3[1],ymm0[2],ymm3[2]4210; CHECK-FAST-PERLANE-NEXT: vmovapd %ymm1, %ymm04211; CHECK-FAST-PERLANE-NEXT: retq4212 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 3, i32 5, i32 0, i32 6>4213 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4214 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec24215 ret <4 x double> %res4216}4217 4218define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask7(<8 x double> %vec, <4 x double> %mask) {4219; CHECK-FAST-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask7:4220; CHECK-FAST: # %bb.0:4221; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [3,5,0,6]4222; CHECK-FAST-NEXT: vxorpd %xmm3, %xmm3, %xmm34223; CHECK-FAST-NEXT: vcmpeqpd %ymm3, %ymm1, %k14224; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}4225; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04226; CHECK-FAST-NEXT: retq4227;4228; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask7:4229; CHECK-FAST-PERLANE: # %bb.0:4230; CHECK-FAST-PERLANE-NEXT: vextractf64x4 $1, %zmm0, %ymm24231; CHECK-FAST-PERLANE-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,3,0,3]4232; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm3, %xmm3, %xmm34233; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm3, %ymm1, %k14234; CHECK-FAST-PERLANE-NEXT: vshufpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],ymm2[1],ymm0[2],ymm2[2]4235; CHECK-FAST-PERLANE-NEXT: retq4236 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 3, i32 5, i32 0, i32 6>4237 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4238 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer4239 ret <4 x double> %res4240}4241define <2 x double> @test_8xdouble_to_2xdouble_perm_mask0(<8 x double> %vec) {4242; CHECK-FAST-LABEL: test_8xdouble_to_2xdouble_perm_mask0:4243; CHECK-FAST: # %bb.0:4244; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} xmm1 = [0,6]4245; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm1, %zmm04246; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm04247; CHECK-FAST-NEXT: vzeroupper4248; CHECK-FAST-NEXT: retq4249;4250; CHECK-FAST-PERLANE-LABEL: test_8xdouble_to_2xdouble_perm_mask0:4251; CHECK-FAST-PERLANE: # %bb.0:4252; CHECK-FAST-PERLANE-NEXT: vextractf32x4 $3, %zmm0, %xmm14253; CHECK-FAST-PERLANE-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]4254; CHECK-FAST-PERLANE-NEXT: vzeroupper4255; CHECK-FAST-PERLANE-NEXT: retq4256 %res = shufflevector <8 x double> %vec, <8 x double> undef, <2 x i32> <i32 0, i32 6>4257 ret <2 x double> %res4258}4259define <2 x double> @test_masked_8xdouble_to_2xdouble_perm_mask0(<8 x double> %vec, <2 x double> %vec2, <2 x double> %mask) {4260; CHECK-FAST-LABEL: test_masked_8xdouble_to_2xdouble_perm_mask0:4261; CHECK-FAST: # %bb.0:4262; CHECK-FAST-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm14263; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} xmm3 = [0,6]4264; CHECK-FAST-NEXT: vxorpd %xmm4, %xmm4, %xmm44265; CHECK-FAST-NEXT: vcmpeqpd %xmm4, %xmm2, %k14266; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}4267; CHECK-FAST-NEXT: vmovapd %xmm1, %xmm04268; CHECK-FAST-NEXT: vzeroupper4269; CHECK-FAST-NEXT: retq4270;4271; CHECK-FAST-PERLANE-LABEL: test_masked_8xdouble_to_2xdouble_perm_mask0:4272; CHECK-FAST-PERLANE: # %bb.0:4273; CHECK-FAST-PERLANE-NEXT: vextractf32x4 $3, %zmm0, %xmm34274; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm4, %xmm4, %xmm44275; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %xmm4, %xmm2, %k14276; CHECK-FAST-PERLANE-NEXT: vunpcklpd {{.*#+}} xmm1 {%k1} = xmm0[0],xmm3[0]4277; CHECK-FAST-PERLANE-NEXT: vmovapd %xmm1, %xmm04278; CHECK-FAST-PERLANE-NEXT: vzeroupper4279; CHECK-FAST-PERLANE-NEXT: retq4280 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <2 x i32> <i32 0, i32 6>4281 %cmp = fcmp oeq <2 x double> %mask, zeroinitializer4282 %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> %vec24283 ret <2 x double> %res4284}4285 4286define <2 x double> @test_masked_z_8xdouble_to_2xdouble_perm_mask0(<8 x double> %vec, <2 x double> %mask) {4287; CHECK-FAST-LABEL: test_masked_z_8xdouble_to_2xdouble_perm_mask0:4288; CHECK-FAST: # %bb.0:4289; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} xmm2 = [0,6]4290; CHECK-FAST-NEXT: vxorpd %xmm3, %xmm3, %xmm34291; CHECK-FAST-NEXT: vcmpeqpd %xmm3, %xmm1, %k14292; CHECK-FAST-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}4293; CHECK-FAST-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm04294; CHECK-FAST-NEXT: vzeroupper4295; CHECK-FAST-NEXT: retq4296;4297; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xdouble_to_2xdouble_perm_mask0:4298; CHECK-FAST-PERLANE: # %bb.0:4299; CHECK-FAST-PERLANE-NEXT: vextractf32x4 $3, %zmm0, %xmm24300; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm3, %xmm3, %xmm34301; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %xmm3, %xmm1, %k14302; CHECK-FAST-PERLANE-NEXT: vunpcklpd {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm2[0]4303; CHECK-FAST-PERLANE-NEXT: vzeroupper4304; CHECK-FAST-PERLANE-NEXT: retq4305 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <2 x i32> <i32 0, i32 6>4306 %cmp = fcmp oeq <2 x double> %mask, zeroinitializer4307 %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> zeroinitializer4308 ret <2 x double> %res4309}4310define <2 x double> @test_masked_8xdouble_to_2xdouble_perm_mask1(<8 x double> %vec, <2 x double> %vec2, <2 x double> %mask) {4311; CHECK-LABEL: test_masked_8xdouble_to_2xdouble_perm_mask1:4312; CHECK: # %bb.0:4313; CHECK-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm14314; CHECK-NEXT: vpmovsxbq {{.*#+}} xmm3 = [3,7]4315; CHECK-NEXT: vxorpd %xmm4, %xmm4, %xmm44316; CHECK-NEXT: vcmpeqpd %xmm4, %xmm2, %k14317; CHECK-NEXT: vpermpd %zmm0, %zmm3, %zmm1 {%k1}4318; CHECK-NEXT: vmovapd %xmm1, %xmm04319; CHECK-NEXT: vzeroupper4320; CHECK-NEXT: retq4321 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <2 x i32> <i32 3, i32 7>4322 %cmp = fcmp oeq <2 x double> %mask, zeroinitializer4323 %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> %vec24324 ret <2 x double> %res4325}4326 4327define <2 x double> @test_masked_z_8xdouble_to_2xdouble_perm_mask1(<8 x double> %vec, <2 x double> %mask) {4328; CHECK-LABEL: test_masked_z_8xdouble_to_2xdouble_perm_mask1:4329; CHECK: # %bb.0:4330; CHECK-NEXT: vpmovsxbq {{.*#+}} xmm2 = [3,7]4331; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm34332; CHECK-NEXT: vcmpeqpd %xmm3, %xmm1, %k14333; CHECK-NEXT: vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z}4334; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm04335; CHECK-NEXT: vzeroupper4336; CHECK-NEXT: retq4337 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <2 x i32> <i32 3, i32 7>4338 %cmp = fcmp oeq <2 x double> %mask, zeroinitializer4339 %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> zeroinitializer4340 ret <2 x double> %res4341}4342define <4 x double> @test_8xdouble_to_4xdouble_perm_mem_mask0(ptr %vp) {4343; CHECK-LABEL: test_8xdouble_to_4xdouble_perm_mem_mask0:4344; CHECK: # %bb.0:4345; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm0 = [1,6,7,2]4346; CHECK-NEXT: vpermpd (%rdi), %zmm0, %zmm04347; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04348; CHECK-NEXT: retq4349 %vec = load <8 x double>, ptr %vp4350 %res = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 1, i32 6, i32 7, i32 2>4351 ret <4 x double> %res4352}4353define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask0(ptr %vp, <4 x double> %vec2, <4 x double> %mask) {4354; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask0:4355; CHECK: # %bb.0:4356; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm04357; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,6,7,2]4358; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm34359; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k14360; CHECK-NEXT: vpermpd (%rdi), %zmm2, %zmm0 {%k1}4361; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04362; CHECK-NEXT: retq4363 %vec = load <8 x double>, ptr %vp4364 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 1, i32 6, i32 7, i32 2>4365 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4366 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec24367 ret <4 x double> %res4368}4369 4370define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask0(ptr %vp, <4 x double> %mask) {4371; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask0:4372; CHECK: # %bb.0:4373; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [1,6,7,2]4374; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm24375; CHECK-NEXT: vcmpeqpd %ymm2, %ymm0, %k14376; CHECK-NEXT: vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z}4377; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04378; CHECK-NEXT: retq4379 %vec = load <8 x double>, ptr %vp4380 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 1, i32 6, i32 7, i32 2>4381 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4382 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer4383 ret <4 x double> %res4384}4385 4386define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask1(ptr %vp, <4 x double> %vec2, <4 x double> %mask) {4387; CHECK-FAST-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask1:4388; CHECK-FAST: # %bb.0:4389; CHECK-FAST-NEXT: vbroadcastsd 32(%rdi), %ymm24390; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm3 = [7,0,6,2]4391; CHECK-FAST-NEXT: vpermi2pd (%rdi), %ymm2, %ymm34392; CHECK-FAST-NEXT: vxorpd %xmm2, %xmm2, %xmm24393; CHECK-FAST-NEXT: vcmpeqpd %ymm2, %ymm1, %k14394; CHECK-FAST-NEXT: vmovapd %ymm3, %ymm0 {%k1}4395; CHECK-FAST-NEXT: retq4396;4397; CHECK-FAST-PERLANE-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask1:4398; CHECK-FAST-PERLANE: # %bb.0:4399; CHECK-FAST-PERLANE-NEXT: vpermpd $236, (%rdi), %ymm2 # ymm2 = mem[0,3,2,3]4400; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm3, %xmm3, %xmm34401; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm3, %ymm1, %k14402; CHECK-FAST-PERLANE-NEXT: vshufpd $1, 32(%rdi){1to4}, %ymm2, %ymm0 {%k1}4403; CHECK-FAST-PERLANE-NEXT: retq4404 %vec = load <8 x double>, ptr %vp4405 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 3, i32 4, i32 2, i32 4>4406 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4407 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec24408 ret <4 x double> %res4409}4410 4411define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask1(ptr %vp, <4 x double> %mask) {4412; CHECK-FAST-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask1:4413; CHECK-FAST: # %bb.0:4414; CHECK-FAST-NEXT: vbroadcastsd 32(%rdi), %ymm24415; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [7,0,6,2]4416; CHECK-FAST-NEXT: vxorpd %xmm3, %xmm3, %xmm34417; CHECK-FAST-NEXT: vcmpeqpd %ymm3, %ymm0, %k14418; CHECK-FAST-NEXT: vpermi2pd (%rdi), %ymm2, %ymm1 {%k1} {z}4419; CHECK-FAST-NEXT: vmovapd %ymm1, %ymm04420; CHECK-FAST-NEXT: retq4421;4422; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask1:4423; CHECK-FAST-PERLANE: # %bb.0:4424; CHECK-FAST-PERLANE-NEXT: vpermpd $236, (%rdi), %ymm1 # ymm1 = mem[0,3,2,3]4425; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm2, %xmm2, %xmm24426; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm2, %ymm0, %k14427; CHECK-FAST-PERLANE-NEXT: vshufpd $1, 32(%rdi){1to4}, %ymm1, %ymm0 {%k1} {z}4428; CHECK-FAST-PERLANE-NEXT: retq4429 %vec = load <8 x double>, ptr %vp4430 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 3, i32 4, i32 2, i32 4>4431 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4432 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer4433 ret <4 x double> %res4434}4435 4436define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask2(ptr %vp, <4 x double> %vec2, <4 x double> %mask) {4437; CHECK-FAST-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask2:4438; CHECK-FAST: # %bb.0:4439; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm04440; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm2 = [1,2,3,4]4441; CHECK-FAST-NEXT: vxorpd %xmm3, %xmm3, %xmm34442; CHECK-FAST-NEXT: vcmpeqpd %ymm3, %ymm1, %k14443; CHECK-FAST-NEXT: vpermpd (%rdi), %zmm2, %zmm0 {%k1}4444; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04445; CHECK-FAST-NEXT: retq4446;4447; CHECK-FAST-PERLANE-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask2:4448; CHECK-FAST-PERLANE: # %bb.0:4449; CHECK-FAST-PERLANE-NEXT: vmovapd (%rdi), %ymm24450; CHECK-FAST-PERLANE-NEXT: vperm2f128 $33, 32(%rdi), %ymm2, %ymm3 # ymm3 = ymm2[2,3],mem[0,1]4451; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm4, %xmm4, %xmm44452; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm4, %ymm1, %k14453; CHECK-FAST-PERLANE-NEXT: vshufpd {{.*#+}} ymm0 {%k1} = ymm2[1],ymm3[0],ymm2[3],ymm3[2]4454; CHECK-FAST-PERLANE-NEXT: retq4455 %vec = load <8 x double>, ptr %vp4456 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 4>4457 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4458 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec24459 ret <4 x double> %res4460}4461 4462define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask2(ptr %vp, <4 x double> %mask) {4463; CHECK-FAST-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask2:4464; CHECK-FAST: # %bb.0:4465; CHECK-FAST-NEXT: vpmovsxbq {{.*#+}} ymm1 = [1,2,3,4]4466; CHECK-FAST-NEXT: vxorpd %xmm2, %xmm2, %xmm24467; CHECK-FAST-NEXT: vcmpeqpd %ymm2, %ymm0, %k14468; CHECK-FAST-NEXT: vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z}4469; CHECK-FAST-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04470; CHECK-FAST-NEXT: retq4471;4472; CHECK-FAST-PERLANE-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask2:4473; CHECK-FAST-PERLANE: # %bb.0:4474; CHECK-FAST-PERLANE-NEXT: vmovapd (%rdi), %ymm14475; CHECK-FAST-PERLANE-NEXT: vperm2f128 $33, 32(%rdi), %ymm1, %ymm2 # ymm2 = ymm1[2,3],mem[0,1]4476; CHECK-FAST-PERLANE-NEXT: vxorpd %xmm3, %xmm3, %xmm34477; CHECK-FAST-PERLANE-NEXT: vcmpeqpd %ymm3, %ymm0, %k14478; CHECK-FAST-PERLANE-NEXT: vshufpd {{.*#+}} ymm0 {%k1} {z} = ymm1[1],ymm2[0],ymm1[3],ymm2[2]4479; CHECK-FAST-PERLANE-NEXT: retq4480 %vec = load <8 x double>, ptr %vp4481 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 4>4482 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4483 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer4484 ret <4 x double> %res4485}4486 4487define <4 x double> @test_8xdouble_to_4xdouble_perm_mem_mask3(ptr %vp) {4488; CHECK-LABEL: test_8xdouble_to_4xdouble_perm_mem_mask3:4489; CHECK: # %bb.0:4490; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm0 = [4,2,1,0]4491; CHECK-NEXT: vpermpd (%rdi), %zmm0, %zmm04492; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04493; CHECK-NEXT: retq4494 %vec = load <8 x double>, ptr %vp4495 %res = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 4, i32 2, i32 1, i32 0>4496 ret <4 x double> %res4497}4498define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask3(ptr %vp, <4 x double> %vec2, <4 x double> %mask) {4499; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask3:4500; CHECK: # %bb.0:4501; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm04502; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [4,2,1,0]4503; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm34504; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k14505; CHECK-NEXT: vpermpd (%rdi), %zmm2, %zmm0 {%k1}4506; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04507; CHECK-NEXT: retq4508 %vec = load <8 x double>, ptr %vp4509 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 4, i32 2, i32 1, i32 0>4510 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4511 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec24512 ret <4 x double> %res4513}4514 4515define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask3(ptr %vp, <4 x double> %mask) {4516; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask3:4517; CHECK: # %bb.0:4518; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [4,2,1,0]4519; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm24520; CHECK-NEXT: vcmpeqpd %ymm2, %ymm0, %k14521; CHECK-NEXT: vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z}4522; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04523; CHECK-NEXT: retq4524 %vec = load <8 x double>, ptr %vp4525 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 4, i32 2, i32 1, i32 0>4526 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4527 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer4528 ret <4 x double> %res4529}4530 4531define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask4(ptr %vp, <4 x double> %vec2, <4 x double> %mask) {4532; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask4:4533; CHECK: # %bb.0:4534; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm04535; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [6,0,5,1]4536; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm34537; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k14538; CHECK-NEXT: vpermpd (%rdi), %zmm2, %zmm0 {%k1}4539; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04540; CHECK-NEXT: retq4541 %vec = load <8 x double>, ptr %vp4542 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 6, i32 0, i32 5, i32 1>4543 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4544 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec24545 ret <4 x double> %res4546}4547 4548define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask4(ptr %vp, <4 x double> %mask) {4549; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask4:4550; CHECK: # %bb.0:4551; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [6,0,5,1]4552; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm24553; CHECK-NEXT: vcmpeqpd %ymm2, %ymm0, %k14554; CHECK-NEXT: vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z}4555; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04556; CHECK-NEXT: retq4557 %vec = load <8 x double>, ptr %vp4558 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 6, i32 0, i32 5, i32 1>4559 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4560 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer4561 ret <4 x double> %res4562}4563 4564define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask5(ptr %vp, <4 x double> %vec2, <4 x double> %mask) {4565; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask5:4566; CHECK: # %bb.0:4567; CHECK-NEXT: vmovapd (%rdi), %ymm24568; CHECK-NEXT: vperm2f128 $33, 32(%rdi), %ymm2, %ymm2 # ymm2 = ymm2[2,3],mem[0,1]4569; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm34570; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k14571; CHECK-NEXT: vshufpd $14, 40(%rdi){1to4}, %ymm2, %ymm0 {%k1}4572; CHECK-NEXT: retq4573 %vec = load <8 x double>, ptr %vp4574 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 2, i32 5, i32 5, i32 5>4575 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4576 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec24577 ret <4 x double> %res4578}4579 4580define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask5(ptr %vp, <4 x double> %mask) {4581; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask5:4582; CHECK: # %bb.0:4583; CHECK-NEXT: vmovapd (%rdi), %ymm14584; CHECK-NEXT: vperm2f128 $33, 32(%rdi), %ymm1, %ymm1 # ymm1 = ymm1[2,3],mem[0,1]4585; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm24586; CHECK-NEXT: vcmpeqpd %ymm2, %ymm0, %k14587; CHECK-NEXT: vshufpd $14, 40(%rdi){1to4}, %ymm1, %ymm0 {%k1} {z}4588; CHECK-NEXT: retq4589 %vec = load <8 x double>, ptr %vp4590 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 2, i32 5, i32 5, i32 5>4591 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4592 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer4593 ret <4 x double> %res4594}4595 4596define <4 x double> @test_8xdouble_to_4xdouble_perm_mem_mask6(ptr %vp) {4597; CHECK-LABEL: test_8xdouble_to_4xdouble_perm_mem_mask6:4598; CHECK: # %bb.0:4599; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm0 = [4,6,0,5]4600; CHECK-NEXT: vpermpd (%rdi), %zmm0, %zmm04601; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04602; CHECK-NEXT: retq4603 %vec = load <8 x double>, ptr %vp4604 %res = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 4, i32 6, i32 0, i32 5>4605 ret <4 x double> %res4606}4607define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask6(ptr %vp, <4 x double> %vec2, <4 x double> %mask) {4608; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask6:4609; CHECK: # %bb.0:4610; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm04611; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm2 = [4,6,0,5]4612; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm34613; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k14614; CHECK-NEXT: vpermpd (%rdi), %zmm2, %zmm0 {%k1}4615; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04616; CHECK-NEXT: retq4617 %vec = load <8 x double>, ptr %vp4618 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 4, i32 6, i32 0, i32 5>4619 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4620 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec24621 ret <4 x double> %res4622}4623 4624define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask6(ptr %vp, <4 x double> %mask) {4625; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask6:4626; CHECK: # %bb.0:4627; CHECK-NEXT: vpmovsxbq {{.*#+}} ymm1 = [4,6,0,5]4628; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm24629; CHECK-NEXT: vcmpeqpd %ymm2, %ymm0, %k14630; CHECK-NEXT: vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z}4631; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm04632; CHECK-NEXT: retq4633 %vec = load <8 x double>, ptr %vp4634 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 4, i32 6, i32 0, i32 5>4635 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4636 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer4637 ret <4 x double> %res4638}4639 4640define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mem_mask7(ptr %vp, <4 x double> %vec2, <4 x double> %mask) {4641; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mem_mask7:4642; CHECK: # %bb.0:4643; CHECK-NEXT: vmovapd (%rdi), %ymm24644; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm34645; CHECK-NEXT: vcmpeqpd %ymm3, %ymm1, %k14646; CHECK-NEXT: vunpcklpd 40(%rdi){1to4}, %ymm2, %ymm0 {%k1}4647; CHECK-NEXT: retq4648 %vec = load <8 x double>, ptr %vp4649 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 5>4650 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4651 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec24652 ret <4 x double> %res4653}4654 4655define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask7(ptr %vp, <4 x double> %mask) {4656; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mem_mask7:4657; CHECK: # %bb.0:4658; CHECK-NEXT: vmovapd (%rdi), %ymm14659; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm24660; CHECK-NEXT: vcmpeqpd %ymm2, %ymm0, %k14661; CHECK-NEXT: vunpcklpd 40(%rdi){1to4}, %ymm1, %ymm0 {%k1} {z}4662; CHECK-NEXT: retq4663 %vec = load <8 x double>, ptr %vp4664 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 5>4665 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer4666 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer4667 ret <4 x double> %res4668}4669 4670define <2 x double> @test_8xdouble_to_2xdouble_perm_mem_mask0(ptr %vp) {4671; CHECK-LABEL: test_8xdouble_to_2xdouble_perm_mem_mask0:4672; CHECK: # %bb.0:4673; CHECK-NEXT: vmovapd (%rdi), %xmm04674; CHECK-NEXT: vshufpd $1, 48(%rdi), %xmm0, %xmm0 # xmm0 = xmm0[1],mem[0]4675; CHECK-NEXT: retq4676 %vec = load <8 x double>, ptr %vp4677 %res = shufflevector <8 x double> %vec, <8 x double> undef, <2 x i32> <i32 1, i32 6>4678 ret <2 x double> %res4679}4680define <2 x double> @test_masked_8xdouble_to_2xdouble_perm_mem_mask0(ptr %vp, <2 x double> %vec2, <2 x double> %mask) {4681; CHECK-LABEL: test_masked_8xdouble_to_2xdouble_perm_mem_mask0:4682; CHECK: # %bb.0:4683; CHECK-NEXT: vmovapd (%rdi), %xmm24684; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm34685; CHECK-NEXT: vcmpeqpd %xmm3, %xmm1, %k14686; CHECK-NEXT: vshufpd $1, 48(%rdi), %xmm2, %xmm0 {%k1} # xmm0 {%k1} = xmm2[1],mem[0]4687; CHECK-NEXT: retq4688 %vec = load <8 x double>, ptr %vp4689 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <2 x i32> <i32 1, i32 6>4690 %cmp = fcmp oeq <2 x double> %mask, zeroinitializer4691 %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> %vec24692 ret <2 x double> %res4693}4694 4695define <2 x double> @test_masked_z_8xdouble_to_2xdouble_perm_mem_mask0(ptr %vp, <2 x double> %mask) {4696; CHECK-LABEL: test_masked_z_8xdouble_to_2xdouble_perm_mem_mask0:4697; CHECK: # %bb.0:4698; CHECK-NEXT: vmovapd (%rdi), %xmm14699; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm24700; CHECK-NEXT: vcmpeqpd %xmm2, %xmm0, %k14701; CHECK-NEXT: vshufpd $1, 48(%rdi), %xmm1, %xmm0 {%k1} {z} # xmm0 {%k1} {z} = xmm1[1],mem[0]4702; CHECK-NEXT: retq4703 %vec = load <8 x double>, ptr %vp4704 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <2 x i32> <i32 1, i32 6>4705 %cmp = fcmp oeq <2 x double> %mask, zeroinitializer4706 %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> zeroinitializer4707 ret <2 x double> %res4708}4709 4710define <2 x double> @test_masked_8xdouble_to_2xdouble_perm_mem_mask1(ptr %vp, <2 x double> %vec2, <2 x double> %mask) {4711; CHECK-LABEL: test_masked_8xdouble_to_2xdouble_perm_mem_mask1:4712; CHECK: # %bb.0:4713; CHECK-NEXT: vmovddup 8(%rdi), %xmm2 # xmm2 = mem[0,0]4714; CHECK-NEXT: vxorpd %xmm3, %xmm3, %xmm34715; CHECK-NEXT: vcmpeqpd %xmm3, %xmm1, %k14716; CHECK-NEXT: vunpcklpd 32(%rdi), %xmm2, %xmm0 {%k1} # xmm0 {%k1} = xmm2[0],mem[0]4717; CHECK-NEXT: retq4718 %vec = load <8 x double>, ptr %vp4719 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <2 x i32> <i32 1, i32 4>4720 %cmp = fcmp oeq <2 x double> %mask, zeroinitializer4721 %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> %vec24722 ret <2 x double> %res4723}4724 4725define <2 x double> @test_masked_z_8xdouble_to_2xdouble_perm_mem_mask1(ptr %vp, <2 x double> %mask) {4726; CHECK-LABEL: test_masked_z_8xdouble_to_2xdouble_perm_mem_mask1:4727; CHECK: # %bb.0:4728; CHECK-NEXT: vmovddup 8(%rdi), %xmm1 # xmm1 = mem[0,0]4729; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm24730; CHECK-NEXT: vcmpeqpd %xmm2, %xmm0, %k14731; CHECK-NEXT: vunpcklpd 32(%rdi), %xmm1, %xmm0 {%k1} {z} # xmm0 {%k1} {z} = xmm1[0],mem[0]4732; CHECK-NEXT: retq4733 %vec = load <8 x double>, ptr %vp4734 %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <2 x i32> <i32 1, i32 4>4735 %cmp = fcmp oeq <2 x double> %mask, zeroinitializer4736 %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> zeroinitializer4737 ret <2 x double> %res4738}4739 4740; PR359774741define void @test_zext_v8i8_to_v8i16(ptr %arg, ptr %arg1) {4742; CHECK-LABEL: test_zext_v8i8_to_v8i16:4743; CHECK: # %bb.0:4744; CHECK-NEXT: vpmovzxbw (%rdi), %xmm0 # xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero4745; CHECK-NEXT: vpsllw $8, %xmm0, %xmm04746; CHECK-NEXT: vmovdqa %xmm0, (%rsi)4747; CHECK-NEXT: retq4748 %tmp2 = load <8 x i8>, ptr %arg4749 %tmp3 = extractelement <8 x i8> %tmp2, i32 04750 %tmp4 = zext i8 %tmp3 to i164751 %tmp5 = insertelement <8 x i16> undef, i16 %tmp4, i32 04752 %tmp6 = extractelement <8 x i8> %tmp2, i32 14753 %tmp7 = zext i8 %tmp6 to i164754 %tmp8 = insertelement <8 x i16> %tmp5, i16 %tmp7, i32 14755 %tmp9 = extractelement <8 x i8> %tmp2, i32 24756 %tmp10 = zext i8 %tmp9 to i164757 %tmp11 = insertelement <8 x i16> %tmp8, i16 %tmp10, i32 24758 %tmp12 = extractelement <8 x i8> %tmp2, i32 34759 %tmp13 = zext i8 %tmp12 to i164760 %tmp14 = insertelement <8 x i16> %tmp11, i16 %tmp13, i32 34761 %tmp15 = extractelement <8 x i8> %tmp2, i32 44762 %tmp16 = zext i8 %tmp15 to i164763 %tmp17 = insertelement <8 x i16> %tmp14, i16 %tmp16, i32 44764 %tmp18 = extractelement <8 x i8> %tmp2, i32 54765 %tmp19 = zext i8 %tmp18 to i164766 %tmp20 = insertelement <8 x i16> %tmp17, i16 %tmp19, i32 54767 %tmp21 = extractelement <8 x i8> %tmp2, i32 64768 %tmp22 = zext i8 %tmp21 to i164769 %tmp23 = insertelement <8 x i16> %tmp20, i16 %tmp22, i32 64770 %tmp24 = extractelement <8 x i8> %tmp2, i32 74771 %tmp25 = zext i8 %tmp24 to i164772 %tmp26 = insertelement <8 x i16> %tmp23, i16 %tmp25, i32 74773 %tmp27 = shl <8 x i16> %tmp26, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>4774 store <8 x i16> %tmp27, ptr %arg14775 ret void4776}4777