1164 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512vl,+avx512dq %s -o - | FileCheck %s3 4define <8 x float> @test_2xfloat_to_8xfloat(<8 x float> %vec) {5; CHECK-LABEL: test_2xfloat_to_8xfloat:6; CHECK: # %bb.0:7; CHECK-NEXT: vbroadcastsd %xmm0, %ymm08; CHECK-NEXT: retq9 %res = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>10 ret <8 x float> %res11}12define <8 x float> @test_masked_2xfloat_to_8xfloat_mask0(<8 x float> %vec, <8 x float> %default, <8 x float> %mask) {13; CHECK-LABEL: test_masked_2xfloat_to_8xfloat_mask0:14; CHECK: # %bb.0:15; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm316; CHECK-NEXT: vcmpeqps %ymm3, %ymm2, %k117; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm1 {%k1} = xmm0[0,1,0,1,0,1,0,1]18; CHECK-NEXT: vmovapd %ymm1, %ymm019; CHECK-NEXT: retq20 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>21 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer22 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %default23 ret <8 x float> %res24}25 26define <8 x float> @test_masked_z_2xfloat_to_8xfloat_mask0(<8 x float> %vec, <8 x float> %mask) {27; CHECK-LABEL: test_masked_z_2xfloat_to_8xfloat_mask0:28; CHECK: # %bb.0:29; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm230; CHECK-NEXT: vcmpeqps %ymm2, %ymm1, %k131; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm0 {%k1} {z} = xmm0[0,1,0,1,0,1,0,1]32; CHECK-NEXT: retq33 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>34 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer35 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer36 ret <8 x float> %res37}38define <8 x float> @test_masked_2xfloat_to_8xfloat_mask1(<8 x float> %vec, <8 x float> %default, <8 x float> %mask) {39; CHECK-LABEL: test_masked_2xfloat_to_8xfloat_mask1:40; CHECK: # %bb.0:41; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm342; CHECK-NEXT: vcmpeqps %ymm3, %ymm2, %k143; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm1 {%k1} = xmm0[0,1,0,1,0,1,0,1]44; CHECK-NEXT: vmovapd %ymm1, %ymm045; CHECK-NEXT: retq46 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>47 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer48 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %default49 ret <8 x float> %res50}51 52define <8 x float> @test_masked_z_2xfloat_to_8xfloat_mask1(<8 x float> %vec, <8 x float> %mask) {53; CHECK-LABEL: test_masked_z_2xfloat_to_8xfloat_mask1:54; CHECK: # %bb.0:55; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm256; CHECK-NEXT: vcmpeqps %ymm2, %ymm1, %k157; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm0 {%k1} {z} = xmm0[0,1,0,1,0,1,0,1]58; CHECK-NEXT: retq59 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>60 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer61 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer62 ret <8 x float> %res63}64define <8 x float> @test_masked_2xfloat_to_8xfloat_mask2(<8 x float> %vec, <8 x float> %default, <8 x float> %mask) {65; CHECK-LABEL: test_masked_2xfloat_to_8xfloat_mask2:66; CHECK: # %bb.0:67; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm368; CHECK-NEXT: vcmpeqps %ymm3, %ymm2, %k169; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm1 {%k1} = xmm0[0,1,0,1,0,1,0,1]70; CHECK-NEXT: vmovapd %ymm1, %ymm071; CHECK-NEXT: retq72 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>73 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer74 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %default75 ret <8 x float> %res76}77 78define <8 x float> @test_masked_z_2xfloat_to_8xfloat_mask2(<8 x float> %vec, <8 x float> %mask) {79; CHECK-LABEL: test_masked_z_2xfloat_to_8xfloat_mask2:80; CHECK: # %bb.0:81; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm282; CHECK-NEXT: vcmpeqps %ymm2, %ymm1, %k183; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm0 {%k1} {z} = xmm0[0,1,0,1,0,1,0,1]84; CHECK-NEXT: retq85 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>86 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer87 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer88 ret <8 x float> %res89}90define <8 x float> @test_masked_2xfloat_to_8xfloat_mask3(<8 x float> %vec, <8 x float> %default, <8 x float> %mask) {91; CHECK-LABEL: test_masked_2xfloat_to_8xfloat_mask3:92; CHECK: # %bb.0:93; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm394; CHECK-NEXT: vcmpeqps %ymm3, %ymm2, %k195; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm1 {%k1} = xmm0[0,1,0,1,0,1,0,1]96; CHECK-NEXT: vmovapd %ymm1, %ymm097; CHECK-NEXT: retq98 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>99 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer100 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %default101 ret <8 x float> %res102}103 104define <8 x float> @test_masked_z_2xfloat_to_8xfloat_mask3(<8 x float> %vec, <8 x float> %mask) {105; CHECK-LABEL: test_masked_z_2xfloat_to_8xfloat_mask3:106; CHECK: # %bb.0:107; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2108; CHECK-NEXT: vcmpeqps %ymm2, %ymm1, %k1109; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm0 {%k1} {z} = xmm0[0,1,0,1,0,1,0,1]110; CHECK-NEXT: retq111 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>112 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer113 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer114 ret <8 x float> %res115}116define <16 x float> @test_2xfloat_to_16xfloat(<16 x float> %vec) {117; CHECK-LABEL: test_2xfloat_to_16xfloat:118; CHECK: # %bb.0:119; CHECK-NEXT: vbroadcastsd %xmm0, %zmm0120; CHECK-NEXT: retq121 %res = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>122 ret <16 x float> %res123}124define <16 x float> @test_masked_2xfloat_to_16xfloat_mask0(<16 x float> %vec, <16 x float> %default, <16 x float> %mask) {125; CHECK-LABEL: test_masked_2xfloat_to_16xfloat_mask0:126; CHECK: # %bb.0:127; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm3128; CHECK-NEXT: vcmpeqps %zmm3, %zmm2, %k1129; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm1 {%k1} = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]130; CHECK-NEXT: vmovapd %zmm1, %zmm0131; CHECK-NEXT: retq132 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>133 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer134 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %default135 ret <16 x float> %res136}137 138define <16 x float> @test_masked_z_2xfloat_to_16xfloat_mask0(<16 x float> %vec, <16 x float> %mask) {139; CHECK-LABEL: test_masked_z_2xfloat_to_16xfloat_mask0:140; CHECK: # %bb.0:141; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2142; CHECK-NEXT: vcmpeqps %zmm2, %zmm1, %k1143; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm0 {%k1} {z} = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]144; CHECK-NEXT: retq145 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>146 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer147 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer148 ret <16 x float> %res149}150define <16 x float> @test_masked_2xfloat_to_16xfloat_mask1(<16 x float> %vec, <16 x float> %default, <16 x float> %mask) {151; CHECK-LABEL: test_masked_2xfloat_to_16xfloat_mask1:152; CHECK: # %bb.0:153; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm3154; CHECK-NEXT: vcmpeqps %zmm3, %zmm2, %k1155; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm1 {%k1} = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]156; CHECK-NEXT: vmovapd %zmm1, %zmm0157; CHECK-NEXT: retq158 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>159 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer160 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %default161 ret <16 x float> %res162}163 164define <16 x float> @test_masked_z_2xfloat_to_16xfloat_mask1(<16 x float> %vec, <16 x float> %mask) {165; CHECK-LABEL: test_masked_z_2xfloat_to_16xfloat_mask1:166; CHECK: # %bb.0:167; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2168; CHECK-NEXT: vcmpeqps %zmm2, %zmm1, %k1169; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm0 {%k1} {z} = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]170; CHECK-NEXT: retq171 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>172 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer173 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer174 ret <16 x float> %res175}176define <16 x float> @test_masked_2xfloat_to_16xfloat_mask2(<16 x float> %vec, <16 x float> %default, <16 x float> %mask) {177; CHECK-LABEL: test_masked_2xfloat_to_16xfloat_mask2:178; CHECK: # %bb.0:179; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm3180; CHECK-NEXT: vcmpeqps %zmm3, %zmm2, %k1181; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm1 {%k1} = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]182; CHECK-NEXT: vmovapd %zmm1, %zmm0183; CHECK-NEXT: retq184 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>185 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer186 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %default187 ret <16 x float> %res188}189 190define <16 x float> @test_masked_z_2xfloat_to_16xfloat_mask2(<16 x float> %vec, <16 x float> %mask) {191; CHECK-LABEL: test_masked_z_2xfloat_to_16xfloat_mask2:192; CHECK: # %bb.0:193; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2194; CHECK-NEXT: vcmpeqps %zmm2, %zmm1, %k1195; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm0 {%k1} {z} = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]196; CHECK-NEXT: retq197 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>198 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer199 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer200 ret <16 x float> %res201}202define <16 x float> @test_masked_2xfloat_to_16xfloat_mask3(<16 x float> %vec, <16 x float> %default, <16 x float> %mask) {203; CHECK-LABEL: test_masked_2xfloat_to_16xfloat_mask3:204; CHECK: # %bb.0:205; CHECK-NEXT: vxorps %xmm3, %xmm3, %xmm3206; CHECK-NEXT: vcmpeqps %zmm3, %zmm2, %k1207; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm1 {%k1} = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]208; CHECK-NEXT: vmovapd %zmm1, %zmm0209; CHECK-NEXT: retq210 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>211 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer212 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %default213 ret <16 x float> %res214}215 216define <16 x float> @test_masked_z_2xfloat_to_16xfloat_mask3(<16 x float> %vec, <16 x float> %mask) {217; CHECK-LABEL: test_masked_z_2xfloat_to_16xfloat_mask3:218; CHECK: # %bb.0:219; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2220; CHECK-NEXT: vcmpeqps %zmm2, %zmm1, %k1221; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm0 {%k1} {z} = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]222; CHECK-NEXT: retq223 %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>224 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer225 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer226 ret <16 x float> %res227}228define <4 x double> @test_2xdouble_to_4xdouble_mem(ptr %vp) {229; CHECK-LABEL: test_2xdouble_to_4xdouble_mem:230; CHECK: # %bb.0:231; CHECK-NEXT: vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]232; CHECK-NEXT: retq233 %vec = load <2 x double>, ptr %vp234 %res = shufflevector <2 x double> %vec, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>235 ret <4 x double> %res236}237define <4 x double> @test_masked_2xdouble_to_4xdouble_mem_mask0(ptr %vp, <4 x double> %default, <4 x double> %mask) {238; CHECK-LABEL: test_masked_2xdouble_to_4xdouble_mem_mask0:239; CHECK: # %bb.0:240; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2241; CHECK-NEXT: vcmpeqpd %ymm2, %ymm1, %k1242; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} ymm0 {%k1} = mem[0,1,0,1]243; CHECK-NEXT: retq244 %vec = load <2 x double>, ptr %vp245 %shuf = shufflevector <2 x double> %vec, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>246 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer247 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %default248 ret <4 x double> %res249}250 251define <4 x double> @test_masked_z_2xdouble_to_4xdouble_mem_mask0(ptr %vp, <4 x double> %mask) {252; CHECK-LABEL: test_masked_z_2xdouble_to_4xdouble_mem_mask0:253; CHECK: # %bb.0:254; CHECK-NEXT: vxorpd %xmm1, %xmm1, %xmm1255; CHECK-NEXT: vcmpeqpd %ymm1, %ymm0, %k1256; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,0,1]257; CHECK-NEXT: retq258 %vec = load <2 x double>, ptr %vp259 %shuf = shufflevector <2 x double> %vec, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>260 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer261 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer262 ret <4 x double> %res263}264define <4 x double> @test_masked_2xdouble_to_4xdouble_mem_mask1(ptr %vp, <4 x double> %default, <4 x double> %mask) {265; CHECK-LABEL: test_masked_2xdouble_to_4xdouble_mem_mask1:266; CHECK: # %bb.0:267; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2268; CHECK-NEXT: vcmpeqpd %ymm2, %ymm1, %k1269; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} ymm0 {%k1} = mem[0,1,0,1]270; CHECK-NEXT: retq271 %vec = load <2 x double>, ptr %vp272 %shuf = shufflevector <2 x double> %vec, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>273 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer274 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %default275 ret <4 x double> %res276}277 278define <4 x double> @test_masked_z_2xdouble_to_4xdouble_mem_mask1(ptr %vp, <4 x double> %mask) {279; CHECK-LABEL: test_masked_z_2xdouble_to_4xdouble_mem_mask1:280; CHECK: # %bb.0:281; CHECK-NEXT: vxorpd %xmm1, %xmm1, %xmm1282; CHECK-NEXT: vcmpeqpd %ymm1, %ymm0, %k1283; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,0,1]284; CHECK-NEXT: retq285 %vec = load <2 x double>, ptr %vp286 %shuf = shufflevector <2 x double> %vec, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>287 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer288 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer289 ret <4 x double> %res290}291define <4 x double> @test_masked_2xdouble_to_4xdouble_mem_mask2(ptr %vp, <4 x double> %default, <4 x double> %mask) {292; CHECK-LABEL: test_masked_2xdouble_to_4xdouble_mem_mask2:293; CHECK: # %bb.0:294; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2295; CHECK-NEXT: vcmpeqpd %ymm2, %ymm1, %k1296; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} ymm0 {%k1} = mem[0,1,0,1]297; CHECK-NEXT: retq298 %vec = load <2 x double>, ptr %vp299 %shuf = shufflevector <2 x double> %vec, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>300 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer301 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %default302 ret <4 x double> %res303}304 305define <4 x double> @test_masked_z_2xdouble_to_4xdouble_mem_mask2(ptr %vp, <4 x double> %mask) {306; CHECK-LABEL: test_masked_z_2xdouble_to_4xdouble_mem_mask2:307; CHECK: # %bb.0:308; CHECK-NEXT: vxorpd %xmm1, %xmm1, %xmm1309; CHECK-NEXT: vcmpeqpd %ymm1, %ymm0, %k1310; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,0,1]311; CHECK-NEXT: retq312 %vec = load <2 x double>, ptr %vp313 %shuf = shufflevector <2 x double> %vec, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>314 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer315 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer316 ret <4 x double> %res317}318define <4 x double> @test_masked_2xdouble_to_4xdouble_mem_mask3(ptr %vp, <4 x double> %default, <4 x double> %mask) {319; CHECK-LABEL: test_masked_2xdouble_to_4xdouble_mem_mask3:320; CHECK: # %bb.0:321; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2322; CHECK-NEXT: vcmpeqpd %ymm2, %ymm1, %k1323; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} ymm0 {%k1} = mem[0,1,0,1]324; CHECK-NEXT: retq325 %vec = load <2 x double>, ptr %vp326 %shuf = shufflevector <2 x double> %vec, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>327 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer328 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %default329 ret <4 x double> %res330}331 332define <4 x double> @test_masked_z_2xdouble_to_4xdouble_mem_mask3(ptr %vp, <4 x double> %mask) {333; CHECK-LABEL: test_masked_z_2xdouble_to_4xdouble_mem_mask3:334; CHECK: # %bb.0:335; CHECK-NEXT: vxorpd %xmm1, %xmm1, %xmm1336; CHECK-NEXT: vcmpeqpd %ymm1, %ymm0, %k1337; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,0,1]338; CHECK-NEXT: retq339 %vec = load <2 x double>, ptr %vp340 %shuf = shufflevector <2 x double> %vec, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>341 %cmp = fcmp oeq <4 x double> %mask, zeroinitializer342 %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer343 ret <4 x double> %res344}345define <8 x double> @test_2xdouble_to_8xdouble_mem(ptr %vp) {346; CHECK-LABEL: test_2xdouble_to_8xdouble_mem:347; CHECK: # %bb.0:348; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]349; CHECK-NEXT: retq350 %vec = load <2 x double>, ptr %vp351 %res = shufflevector <2 x double> %vec, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>352 ret <8 x double> %res353}354define <8 x double> @test_masked_2xdouble_to_8xdouble_mem_mask0(ptr %vp, <8 x double> %default, <8 x double> %mask) {355; CHECK-LABEL: test_masked_2xdouble_to_8xdouble_mem_mask0:356; CHECK: # %bb.0:357; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2358; CHECK-NEXT: vcmpeqpd %zmm2, %zmm1, %k1359; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} zmm0 {%k1} = mem[0,1,0,1,0,1,0,1]360; CHECK-NEXT: retq361 %vec = load <2 x double>, ptr %vp362 %shuf = shufflevector <2 x double> %vec, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>363 %cmp = fcmp oeq <8 x double> %mask, zeroinitializer364 %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %default365 ret <8 x double> %res366}367 368define <8 x double> @test_masked_z_2xdouble_to_8xdouble_mem_mask0(ptr %vp, <8 x double> %mask) {369; CHECK-LABEL: test_masked_z_2xdouble_to_8xdouble_mem_mask0:370; CHECK: # %bb.0:371; CHECK-NEXT: vxorpd %xmm1, %xmm1, %xmm1372; CHECK-NEXT: vcmpeqpd %zmm1, %zmm0, %k1373; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1]374; CHECK-NEXT: retq375 %vec = load <2 x double>, ptr %vp376 %shuf = shufflevector <2 x double> %vec, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>377 %cmp = fcmp oeq <8 x double> %mask, zeroinitializer378 %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer379 ret <8 x double> %res380}381define <8 x double> @test_masked_2xdouble_to_8xdouble_mem_mask1(ptr %vp, <8 x double> %default, <8 x double> %mask) {382; CHECK-LABEL: test_masked_2xdouble_to_8xdouble_mem_mask1:383; CHECK: # %bb.0:384; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2385; CHECK-NEXT: vcmpeqpd %zmm2, %zmm1, %k1386; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} zmm0 {%k1} = mem[0,1,0,1,0,1,0,1]387; CHECK-NEXT: retq388 %vec = load <2 x double>, ptr %vp389 %shuf = shufflevector <2 x double> %vec, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>390 %cmp = fcmp oeq <8 x double> %mask, zeroinitializer391 %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %default392 ret <8 x double> %res393}394 395define <8 x double> @test_masked_z_2xdouble_to_8xdouble_mem_mask1(ptr %vp, <8 x double> %mask) {396; CHECK-LABEL: test_masked_z_2xdouble_to_8xdouble_mem_mask1:397; CHECK: # %bb.0:398; CHECK-NEXT: vxorpd %xmm1, %xmm1, %xmm1399; CHECK-NEXT: vcmpeqpd %zmm1, %zmm0, %k1400; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1]401; CHECK-NEXT: retq402 %vec = load <2 x double>, ptr %vp403 %shuf = shufflevector <2 x double> %vec, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>404 %cmp = fcmp oeq <8 x double> %mask, zeroinitializer405 %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer406 ret <8 x double> %res407}408define <8 x double> @test_masked_2xdouble_to_8xdouble_mem_mask2(ptr %vp, <8 x double> %default, <8 x double> %mask) {409; CHECK-LABEL: test_masked_2xdouble_to_8xdouble_mem_mask2:410; CHECK: # %bb.0:411; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2412; CHECK-NEXT: vcmpeqpd %zmm2, %zmm1, %k1413; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} zmm0 {%k1} = mem[0,1,0,1,0,1,0,1]414; CHECK-NEXT: retq415 %vec = load <2 x double>, ptr %vp416 %shuf = shufflevector <2 x double> %vec, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>417 %cmp = fcmp oeq <8 x double> %mask, zeroinitializer418 %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %default419 ret <8 x double> %res420}421 422define <8 x double> @test_masked_z_2xdouble_to_8xdouble_mem_mask2(ptr %vp, <8 x double> %mask) {423; CHECK-LABEL: test_masked_z_2xdouble_to_8xdouble_mem_mask2:424; CHECK: # %bb.0:425; CHECK-NEXT: vxorpd %xmm1, %xmm1, %xmm1426; CHECK-NEXT: vcmpeqpd %zmm1, %zmm0, %k1427; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1]428; CHECK-NEXT: retq429 %vec = load <2 x double>, ptr %vp430 %shuf = shufflevector <2 x double> %vec, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>431 %cmp = fcmp oeq <8 x double> %mask, zeroinitializer432 %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer433 ret <8 x double> %res434}435define <8 x double> @test_masked_2xdouble_to_8xdouble_mem_mask3(ptr %vp, <8 x double> %default, <8 x double> %mask) {436; CHECK-LABEL: test_masked_2xdouble_to_8xdouble_mem_mask3:437; CHECK: # %bb.0:438; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2439; CHECK-NEXT: vcmpeqpd %zmm2, %zmm1, %k1440; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} zmm0 {%k1} = mem[0,1,0,1,0,1,0,1]441; CHECK-NEXT: retq442 %vec = load <2 x double>, ptr %vp443 %shuf = shufflevector <2 x double> %vec, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>444 %cmp = fcmp oeq <8 x double> %mask, zeroinitializer445 %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %default446 ret <8 x double> %res447}448 449define <8 x double> @test_masked_z_2xdouble_to_8xdouble_mem_mask3(ptr %vp, <8 x double> %mask) {450; CHECK-LABEL: test_masked_z_2xdouble_to_8xdouble_mem_mask3:451; CHECK: # %bb.0:452; CHECK-NEXT: vxorpd %xmm1, %xmm1, %xmm1453; CHECK-NEXT: vcmpeqpd %zmm1, %zmm0, %k1454; CHECK-NEXT: vbroadcastf64x2 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1]455; CHECK-NEXT: retq456 %vec = load <2 x double>, ptr %vp457 %shuf = shufflevector <2 x double> %vec, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>458 %cmp = fcmp oeq <8 x double> %mask, zeroinitializer459 %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer460 ret <8 x double> %res461}462define <8 x double> @test_4xdouble_to_8xdouble_mem(ptr %vp) {463; CHECK-LABEL: test_4xdouble_to_8xdouble_mem:464; CHECK: # %bb.0:465; CHECK-NEXT: vbroadcastf64x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3]466; CHECK-NEXT: retq467 %vec = load <4 x double>, ptr %vp468 %res = shufflevector <4 x double> %vec, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>469 ret <8 x double> %res470}471define <8 x double> @test_masked_4xdouble_to_8xdouble_mem_mask0(ptr %vp, <8 x double> %default, <8 x double> %mask) {472; CHECK-LABEL: test_masked_4xdouble_to_8xdouble_mem_mask0:473; CHECK: # %bb.0:474; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2475; CHECK-NEXT: vcmpeqpd %zmm2, %zmm1, %k1476; CHECK-NEXT: vbroadcastf64x4 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,0,1,2,3]477; CHECK-NEXT: retq478 %vec = load <4 x double>, ptr %vp479 %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>480 %cmp = fcmp oeq <8 x double> %mask, zeroinitializer481 %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %default482 ret <8 x double> %res483}484 485define <8 x double> @test_masked_z_4xdouble_to_8xdouble_mem_mask0(ptr %vp, <8 x double> %mask) {486; CHECK-LABEL: test_masked_z_4xdouble_to_8xdouble_mem_mask0:487; CHECK: # %bb.0:488; CHECK-NEXT: vxorpd %xmm1, %xmm1, %xmm1489; CHECK-NEXT: vcmpeqpd %zmm1, %zmm0, %k1490; CHECK-NEXT: vbroadcastf64x4 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3]491; CHECK-NEXT: retq492 %vec = load <4 x double>, ptr %vp493 %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>494 %cmp = fcmp oeq <8 x double> %mask, zeroinitializer495 %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer496 ret <8 x double> %res497}498define <8 x double> @test_masked_4xdouble_to_8xdouble_mem_mask1(ptr %vp, <8 x double> %default, <8 x double> %mask) {499; CHECK-LABEL: test_masked_4xdouble_to_8xdouble_mem_mask1:500; CHECK: # %bb.0:501; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2502; CHECK-NEXT: vcmpeqpd %zmm2, %zmm1, %k1503; CHECK-NEXT: vbroadcastf64x4 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,0,1,2,3]504; CHECK-NEXT: retq505 %vec = load <4 x double>, ptr %vp506 %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>507 %cmp = fcmp oeq <8 x double> %mask, zeroinitializer508 %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %default509 ret <8 x double> %res510}511 512define <8 x double> @test_masked_z_4xdouble_to_8xdouble_mem_mask1(ptr %vp, <8 x double> %mask) {513; CHECK-LABEL: test_masked_z_4xdouble_to_8xdouble_mem_mask1:514; CHECK: # %bb.0:515; CHECK-NEXT: vxorpd %xmm1, %xmm1, %xmm1516; CHECK-NEXT: vcmpeqpd %zmm1, %zmm0, %k1517; CHECK-NEXT: vbroadcastf64x4 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3]518; CHECK-NEXT: retq519 %vec = load <4 x double>, ptr %vp520 %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>521 %cmp = fcmp oeq <8 x double> %mask, zeroinitializer522 %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer523 ret <8 x double> %res524}525define <8 x double> @test_masked_4xdouble_to_8xdouble_mem_mask2(ptr %vp, <8 x double> %default, <8 x double> %mask) {526; CHECK-LABEL: test_masked_4xdouble_to_8xdouble_mem_mask2:527; CHECK: # %bb.0:528; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2529; CHECK-NEXT: vcmpeqpd %zmm2, %zmm1, %k1530; CHECK-NEXT: vbroadcastf64x4 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,0,1,2,3]531; CHECK-NEXT: retq532 %vec = load <4 x double>, ptr %vp533 %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>534 %cmp = fcmp oeq <8 x double> %mask, zeroinitializer535 %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %default536 ret <8 x double> %res537}538 539define <8 x double> @test_masked_z_4xdouble_to_8xdouble_mem_mask2(ptr %vp, <8 x double> %mask) {540; CHECK-LABEL: test_masked_z_4xdouble_to_8xdouble_mem_mask2:541; CHECK: # %bb.0:542; CHECK-NEXT: vxorpd %xmm1, %xmm1, %xmm1543; CHECK-NEXT: vcmpeqpd %zmm1, %zmm0, %k1544; CHECK-NEXT: vbroadcastf64x4 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3]545; CHECK-NEXT: retq546 %vec = load <4 x double>, ptr %vp547 %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>548 %cmp = fcmp oeq <8 x double> %mask, zeroinitializer549 %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer550 ret <8 x double> %res551}552define <8 x double> @test_masked_4xdouble_to_8xdouble_mem_mask3(ptr %vp, <8 x double> %default, <8 x double> %mask) {553; CHECK-LABEL: test_masked_4xdouble_to_8xdouble_mem_mask3:554; CHECK: # %bb.0:555; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2556; CHECK-NEXT: vcmpeqpd %zmm2, %zmm1, %k1557; CHECK-NEXT: vbroadcastf64x4 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,0,1,2,3]558; CHECK-NEXT: retq559 %vec = load <4 x double>, ptr %vp560 %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>561 %cmp = fcmp oeq <8 x double> %mask, zeroinitializer562 %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %default563 ret <8 x double> %res564}565 566define <8 x double> @test_masked_z_4xdouble_to_8xdouble_mem_mask3(ptr %vp, <8 x double> %mask) {567; CHECK-LABEL: test_masked_z_4xdouble_to_8xdouble_mem_mask3:568; CHECK: # %bb.0:569; CHECK-NEXT: vxorpd %xmm1, %xmm1, %xmm1570; CHECK-NEXT: vcmpeqpd %zmm1, %zmm0, %k1571; CHECK-NEXT: vbroadcastf64x4 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3]572; CHECK-NEXT: retq573 %vec = load <4 x double>, ptr %vp574 %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>575 %cmp = fcmp oeq <8 x double> %mask, zeroinitializer576 %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer577 ret <8 x double> %res578}579define <8 x float> @test_2xfloat_to_8xfloat_mem(ptr %vp) {580; CHECK-LABEL: test_2xfloat_to_8xfloat_mem:581; CHECK: # %bb.0:582; CHECK-NEXT: vbroadcastsd (%rdi), %ymm0583; CHECK-NEXT: retq584 %vec = load <2 x float>, ptr %vp585 %res = shufflevector <2 x float> %vec, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>586 ret <8 x float> %res587}588define <8 x float> @test_masked_2xfloat_to_8xfloat_mem_mask0(ptr %vp, <8 x float> %default, <8 x float> %mask) {589; CHECK-LABEL: test_masked_2xfloat_to_8xfloat_mem_mask0:590; CHECK: # %bb.0:591; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2592; CHECK-NEXT: vcmpeqps %ymm2, %ymm1, %k1593; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm0 {%k1} = mem[0,1,0,1,0,1,0,1]594; CHECK-NEXT: retq595 %vec = load <2 x float>, ptr %vp596 %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>597 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer598 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %default599 ret <8 x float> %res600}601 602define <8 x float> @test_masked_z_2xfloat_to_8xfloat_mem_mask0(ptr %vp, <8 x float> %mask) {603; CHECK-LABEL: test_masked_z_2xfloat_to_8xfloat_mem_mask0:604; CHECK: # %bb.0:605; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1606; CHECK-NEXT: vcmpeqps %ymm1, %ymm0, %k1607; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1]608; CHECK-NEXT: retq609 %vec = load <2 x float>, ptr %vp610 %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>611 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer612 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer613 ret <8 x float> %res614}615define <8 x float> @test_masked_2xfloat_to_8xfloat_mem_mask1(ptr %vp, <8 x float> %default, <8 x float> %mask) {616; CHECK-LABEL: test_masked_2xfloat_to_8xfloat_mem_mask1:617; CHECK: # %bb.0:618; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2619; CHECK-NEXT: vcmpeqps %ymm2, %ymm1, %k1620; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm0 {%k1} = mem[0,1,0,1,0,1,0,1]621; CHECK-NEXT: retq622 %vec = load <2 x float>, ptr %vp623 %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>624 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer625 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %default626 ret <8 x float> %res627}628 629define <8 x float> @test_masked_z_2xfloat_to_8xfloat_mem_mask1(ptr %vp, <8 x float> %mask) {630; CHECK-LABEL: test_masked_z_2xfloat_to_8xfloat_mem_mask1:631; CHECK: # %bb.0:632; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1633; CHECK-NEXT: vcmpeqps %ymm1, %ymm0, %k1634; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1]635; CHECK-NEXT: retq636 %vec = load <2 x float>, ptr %vp637 %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>638 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer639 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer640 ret <8 x float> %res641}642define <8 x float> @test_masked_2xfloat_to_8xfloat_mem_mask2(ptr %vp, <8 x float> %default, <8 x float> %mask) {643; CHECK-LABEL: test_masked_2xfloat_to_8xfloat_mem_mask2:644; CHECK: # %bb.0:645; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2646; CHECK-NEXT: vcmpeqps %ymm2, %ymm1, %k1647; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm0 {%k1} = mem[0,1,0,1,0,1,0,1]648; CHECK-NEXT: retq649 %vec = load <2 x float>, ptr %vp650 %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>651 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer652 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %default653 ret <8 x float> %res654}655 656define <8 x float> @test_masked_z_2xfloat_to_8xfloat_mem_mask2(ptr %vp, <8 x float> %mask) {657; CHECK-LABEL: test_masked_z_2xfloat_to_8xfloat_mem_mask2:658; CHECK: # %bb.0:659; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1660; CHECK-NEXT: vcmpeqps %ymm1, %ymm0, %k1661; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1]662; CHECK-NEXT: retq663 %vec = load <2 x float>, ptr %vp664 %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>665 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer666 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer667 ret <8 x float> %res668}669define <8 x float> @test_masked_2xfloat_to_8xfloat_mem_mask3(ptr %vp, <8 x float> %default, <8 x float> %mask) {670; CHECK-LABEL: test_masked_2xfloat_to_8xfloat_mem_mask3:671; CHECK: # %bb.0:672; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2673; CHECK-NEXT: vcmpeqps %ymm2, %ymm1, %k1674; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm0 {%k1} = mem[0,1,0,1,0,1,0,1]675; CHECK-NEXT: retq676 %vec = load <2 x float>, ptr %vp677 %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>678 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer679 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %default680 ret <8 x float> %res681}682 683define <8 x float> @test_masked_z_2xfloat_to_8xfloat_mem_mask3(ptr %vp, <8 x float> %mask) {684; CHECK-LABEL: test_masked_z_2xfloat_to_8xfloat_mem_mask3:685; CHECK: # %bb.0:686; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1687; CHECK-NEXT: vcmpeqps %ymm1, %ymm0, %k1688; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1]689; CHECK-NEXT: retq690 %vec = load <2 x float>, ptr %vp691 %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>692 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer693 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer694 ret <8 x float> %res695}696define <16 x float> @test_2xfloat_to_16xfloat_mem(ptr %vp) {697; CHECK-LABEL: test_2xfloat_to_16xfloat_mem:698; CHECK: # %bb.0:699; CHECK-NEXT: vbroadcastsd (%rdi), %zmm0700; CHECK-NEXT: retq701 %vec = load <2 x float>, ptr %vp702 %res = shufflevector <2 x float> %vec, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>703 ret <16 x float> %res704}705define <16 x float> @test_masked_2xfloat_to_16xfloat_mem_mask0(ptr %vp, <16 x float> %default, <16 x float> %mask) {706; CHECK-LABEL: test_masked_2xfloat_to_16xfloat_mem_mask0:707; CHECK: # %bb.0:708; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2709; CHECK-NEXT: vcmpeqps %zmm2, %zmm1, %k1710; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm0 {%k1} = mem[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]711; CHECK-NEXT: retq712 %vec = load <2 x float>, ptr %vp713 %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>714 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer715 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %default716 ret <16 x float> %res717}718 719define <16 x float> @test_masked_z_2xfloat_to_16xfloat_mem_mask0(ptr %vp, <16 x float> %mask) {720; CHECK-LABEL: test_masked_z_2xfloat_to_16xfloat_mem_mask0:721; CHECK: # %bb.0:722; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1723; CHECK-NEXT: vcmpeqps %zmm1, %zmm0, %k1724; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]725; CHECK-NEXT: retq726 %vec = load <2 x float>, ptr %vp727 %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>728 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer729 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer730 ret <16 x float> %res731}732define <16 x float> @test_masked_2xfloat_to_16xfloat_mem_mask1(ptr %vp, <16 x float> %default, <16 x float> %mask) {733; CHECK-LABEL: test_masked_2xfloat_to_16xfloat_mem_mask1:734; CHECK: # %bb.0:735; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2736; CHECK-NEXT: vcmpeqps %zmm2, %zmm1, %k1737; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm0 {%k1} = mem[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]738; CHECK-NEXT: retq739 %vec = load <2 x float>, ptr %vp740 %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>741 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer742 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %default743 ret <16 x float> %res744}745 746define <16 x float> @test_masked_z_2xfloat_to_16xfloat_mem_mask1(ptr %vp, <16 x float> %mask) {747; CHECK-LABEL: test_masked_z_2xfloat_to_16xfloat_mem_mask1:748; CHECK: # %bb.0:749; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1750; CHECK-NEXT: vcmpeqps %zmm1, %zmm0, %k1751; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]752; CHECK-NEXT: retq753 %vec = load <2 x float>, ptr %vp754 %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>755 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer756 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer757 ret <16 x float> %res758}759define <16 x float> @test_masked_2xfloat_to_16xfloat_mem_mask2(ptr %vp, <16 x float> %default, <16 x float> %mask) {760; CHECK-LABEL: test_masked_2xfloat_to_16xfloat_mem_mask2:761; CHECK: # %bb.0:762; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2763; CHECK-NEXT: vcmpeqps %zmm2, %zmm1, %k1764; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm0 {%k1} = mem[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]765; CHECK-NEXT: retq766 %vec = load <2 x float>, ptr %vp767 %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>768 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer769 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %default770 ret <16 x float> %res771}772 773define <16 x float> @test_masked_z_2xfloat_to_16xfloat_mem_mask2(ptr %vp, <16 x float> %mask) {774; CHECK-LABEL: test_masked_z_2xfloat_to_16xfloat_mem_mask2:775; CHECK: # %bb.0:776; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1777; CHECK-NEXT: vcmpeqps %zmm1, %zmm0, %k1778; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]779; CHECK-NEXT: retq780 %vec = load <2 x float>, ptr %vp781 %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>782 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer783 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer784 ret <16 x float> %res785}786define <16 x float> @test_masked_2xfloat_to_16xfloat_mem_mask3(ptr %vp, <16 x float> %default, <16 x float> %mask) {787; CHECK-LABEL: test_masked_2xfloat_to_16xfloat_mem_mask3:788; CHECK: # %bb.0:789; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2790; CHECK-NEXT: vcmpeqps %zmm2, %zmm1, %k1791; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm0 {%k1} = mem[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]792; CHECK-NEXT: retq793 %vec = load <2 x float>, ptr %vp794 %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>795 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer796 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %default797 ret <16 x float> %res798}799 800define <16 x float> @test_masked_z_2xfloat_to_16xfloat_mem_mask3(ptr %vp, <16 x float> %mask) {801; CHECK-LABEL: test_masked_z_2xfloat_to_16xfloat_mem_mask3:802; CHECK: # %bb.0:803; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1804; CHECK-NEXT: vcmpeqps %zmm1, %zmm0, %k1805; CHECK-NEXT: vbroadcastf32x2 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]806; CHECK-NEXT: retq807 %vec = load <2 x float>, ptr %vp808 %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>809 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer810 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer811 ret <16 x float> %res812}813define <8 x float> @test_4xfloat_to_8xfloat_mem(ptr %vp) {814; CHECK-LABEL: test_4xfloat_to_8xfloat_mem:815; CHECK: # %bb.0:816; CHECK-NEXT: vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]817; CHECK-NEXT: retq818 %vec = load <4 x float>, ptr %vp819 %res = shufflevector <4 x float> %vec, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>820 ret <8 x float> %res821}822define <8 x float> @test_masked_4xfloat_to_8xfloat_mem_mask0(ptr %vp, <8 x float> %default, <8 x float> %mask) {823; CHECK-LABEL: test_masked_4xfloat_to_8xfloat_mem_mask0:824; CHECK: # %bb.0:825; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2826; CHECK-NEXT: vcmpeqps %ymm2, %ymm1, %k1827; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} ymm0 {%k1} = mem[0,1,2,3,0,1,2,3]828; CHECK-NEXT: retq829 %vec = load <4 x float>, ptr %vp830 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>831 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer832 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %default833 ret <8 x float> %res834}835 836define <8 x float> @test_masked_z_4xfloat_to_8xfloat_mem_mask0(ptr %vp, <8 x float> %mask) {837; CHECK-LABEL: test_masked_z_4xfloat_to_8xfloat_mem_mask0:838; CHECK: # %bb.0:839; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1840; CHECK-NEXT: vcmpeqps %ymm1, %ymm0, %k1841; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3]842; CHECK-NEXT: retq843 %vec = load <4 x float>, ptr %vp844 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>845 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer846 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer847 ret <8 x float> %res848}849define <8 x float> @test_masked_4xfloat_to_8xfloat_mem_mask1(ptr %vp, <8 x float> %default, <8 x float> %mask) {850; CHECK-LABEL: test_masked_4xfloat_to_8xfloat_mem_mask1:851; CHECK: # %bb.0:852; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2853; CHECK-NEXT: vcmpeqps %ymm2, %ymm1, %k1854; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} ymm0 {%k1} = mem[0,1,2,3,0,1,2,3]855; CHECK-NEXT: retq856 %vec = load <4 x float>, ptr %vp857 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>858 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer859 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %default860 ret <8 x float> %res861}862 863define <8 x float> @test_masked_z_4xfloat_to_8xfloat_mem_mask1(ptr %vp, <8 x float> %mask) {864; CHECK-LABEL: test_masked_z_4xfloat_to_8xfloat_mem_mask1:865; CHECK: # %bb.0:866; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1867; CHECK-NEXT: vcmpeqps %ymm1, %ymm0, %k1868; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3]869; CHECK-NEXT: retq870 %vec = load <4 x float>, ptr %vp871 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>872 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer873 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer874 ret <8 x float> %res875}876define <8 x float> @test_masked_4xfloat_to_8xfloat_mem_mask2(ptr %vp, <8 x float> %default, <8 x float> %mask) {877; CHECK-LABEL: test_masked_4xfloat_to_8xfloat_mem_mask2:878; CHECK: # %bb.0:879; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2880; CHECK-NEXT: vcmpeqps %ymm2, %ymm1, %k1881; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} ymm0 {%k1} = mem[0,1,2,3,0,1,2,3]882; CHECK-NEXT: retq883 %vec = load <4 x float>, ptr %vp884 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>885 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer886 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %default887 ret <8 x float> %res888}889 890define <8 x float> @test_masked_z_4xfloat_to_8xfloat_mem_mask2(ptr %vp, <8 x float> %mask) {891; CHECK-LABEL: test_masked_z_4xfloat_to_8xfloat_mem_mask2:892; CHECK: # %bb.0:893; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1894; CHECK-NEXT: vcmpeqps %ymm1, %ymm0, %k1895; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3]896; CHECK-NEXT: retq897 %vec = load <4 x float>, ptr %vp898 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>899 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer900 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer901 ret <8 x float> %res902}903define <8 x float> @test_masked_4xfloat_to_8xfloat_mem_mask3(ptr %vp, <8 x float> %default, <8 x float> %mask) {904; CHECK-LABEL: test_masked_4xfloat_to_8xfloat_mem_mask3:905; CHECK: # %bb.0:906; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2907; CHECK-NEXT: vcmpeqps %ymm2, %ymm1, %k1908; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} ymm0 {%k1} = mem[0,1,2,3,0,1,2,3]909; CHECK-NEXT: retq910 %vec = load <4 x float>, ptr %vp911 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>912 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer913 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %default914 ret <8 x float> %res915}916 917define <8 x float> @test_masked_z_4xfloat_to_8xfloat_mem_mask3(ptr %vp, <8 x float> %mask) {918; CHECK-LABEL: test_masked_z_4xfloat_to_8xfloat_mem_mask3:919; CHECK: # %bb.0:920; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1921; CHECK-NEXT: vcmpeqps %ymm1, %ymm0, %k1922; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3]923; CHECK-NEXT: retq924 %vec = load <4 x float>, ptr %vp925 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>926 %cmp = fcmp oeq <8 x float> %mask, zeroinitializer927 %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer928 ret <8 x float> %res929}930define <16 x float> @test_4xfloat_to_16xfloat_mem(ptr %vp) {931; CHECK-LABEL: test_4xfloat_to_16xfloat_mem:932; CHECK: # %bb.0:933; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]934; CHECK-NEXT: retq935 %vec = load <4 x float>, ptr %vp936 %res = shufflevector <4 x float> %vec, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>937 ret <16 x float> %res938}939define <16 x float> @test_masked_4xfloat_to_16xfloat_mem_mask0(ptr %vp, <16 x float> %default, <16 x float> %mask) {940; CHECK-LABEL: test_masked_4xfloat_to_16xfloat_mem_mask0:941; CHECK: # %bb.0:942; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2943; CHECK-NEXT: vcmpeqps %zmm2, %zmm1, %k1944; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]945; CHECK-NEXT: retq946 %vec = load <4 x float>, ptr %vp947 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>948 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer949 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %default950 ret <16 x float> %res951}952 953define <16 x float> @test_masked_z_4xfloat_to_16xfloat_mem_mask0(ptr %vp, <16 x float> %mask) {954; CHECK-LABEL: test_masked_z_4xfloat_to_16xfloat_mem_mask0:955; CHECK: # %bb.0:956; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1957; CHECK-NEXT: vcmpeqps %zmm1, %zmm0, %k1958; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]959; CHECK-NEXT: retq960 %vec = load <4 x float>, ptr %vp961 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>962 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer963 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer964 ret <16 x float> %res965}966define <16 x float> @test_masked_4xfloat_to_16xfloat_mem_mask1(ptr %vp, <16 x float> %default, <16 x float> %mask) {967; CHECK-LABEL: test_masked_4xfloat_to_16xfloat_mem_mask1:968; CHECK: # %bb.0:969; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2970; CHECK-NEXT: vcmpeqps %zmm2, %zmm1, %k1971; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]972; CHECK-NEXT: retq973 %vec = load <4 x float>, ptr %vp974 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>975 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer976 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %default977 ret <16 x float> %res978}979 980define <16 x float> @test_masked_z_4xfloat_to_16xfloat_mem_mask1(ptr %vp, <16 x float> %mask) {981; CHECK-LABEL: test_masked_z_4xfloat_to_16xfloat_mem_mask1:982; CHECK: # %bb.0:983; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1984; CHECK-NEXT: vcmpeqps %zmm1, %zmm0, %k1985; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]986; CHECK-NEXT: retq987 %vec = load <4 x float>, ptr %vp988 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>989 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer990 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer991 ret <16 x float> %res992}993define <16 x float> @test_masked_4xfloat_to_16xfloat_mem_mask2(ptr %vp, <16 x float> %default, <16 x float> %mask) {994; CHECK-LABEL: test_masked_4xfloat_to_16xfloat_mem_mask2:995; CHECK: # %bb.0:996; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2997; CHECK-NEXT: vcmpeqps %zmm2, %zmm1, %k1998; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]999; CHECK-NEXT: retq1000 %vec = load <4 x float>, ptr %vp1001 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>1002 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer1003 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %default1004 ret <16 x float> %res1005}1006 1007define <16 x float> @test_masked_z_4xfloat_to_16xfloat_mem_mask2(ptr %vp, <16 x float> %mask) {1008; CHECK-LABEL: test_masked_z_4xfloat_to_16xfloat_mem_mask2:1009; CHECK: # %bb.0:1010; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm11011; CHECK-NEXT: vcmpeqps %zmm1, %zmm0, %k11012; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]1013; CHECK-NEXT: retq1014 %vec = load <4 x float>, ptr %vp1015 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>1016 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer1017 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer1018 ret <16 x float> %res1019}1020define <16 x float> @test_masked_4xfloat_to_16xfloat_mem_mask3(ptr %vp, <16 x float> %default, <16 x float> %mask) {1021; CHECK-LABEL: test_masked_4xfloat_to_16xfloat_mem_mask3:1022; CHECK: # %bb.0:1023; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm21024; CHECK-NEXT: vcmpeqps %zmm2, %zmm1, %k11025; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]1026; CHECK-NEXT: retq1027 %vec = load <4 x float>, ptr %vp1028 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>1029 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer1030 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %default1031 ret <16 x float> %res1032}1033 1034define <16 x float> @test_masked_z_4xfloat_to_16xfloat_mem_mask3(ptr %vp, <16 x float> %mask) {1035; CHECK-LABEL: test_masked_z_4xfloat_to_16xfloat_mem_mask3:1036; CHECK: # %bb.0:1037; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm11038; CHECK-NEXT: vcmpeqps %zmm1, %zmm0, %k11039; CHECK-NEXT: vbroadcastf32x4 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]1040; CHECK-NEXT: retq1041 %vec = load <4 x float>, ptr %vp1042 %shuf = shufflevector <4 x float> %vec, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>1043 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer1044 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer1045 ret <16 x float> %res1046}1047define <16 x float> @test_8xfloat_to_16xfloat_mem(ptr %vp) {1048; CHECK-LABEL: test_8xfloat_to_16xfloat_mem:1049; CHECK: # %bb.0:1050; CHECK-NEXT: vbroadcastf64x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3]1051; CHECK-NEXT: retq1052 %vec = load <8 x float>, ptr %vp1053 %res = shufflevector <8 x float> %vec, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>1054 ret <16 x float> %res1055}1056define <16 x float> @test_masked_8xfloat_to_16xfloat_mem_mask0(ptr %vp, <16 x float> %default, <16 x float> %mask) {1057; CHECK-LABEL: test_masked_8xfloat_to_16xfloat_mem_mask0:1058; CHECK: # %bb.0:1059; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm21060; CHECK-NEXT: vcmpeqps %zmm2, %zmm1, %k11061; CHECK-NEXT: vbroadcastf32x8 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]1062; CHECK-NEXT: retq1063 %vec = load <8 x float>, ptr %vp1064 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>1065 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer1066 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %default1067 ret <16 x float> %res1068}1069 1070define <16 x float> @test_masked_z_8xfloat_to_16xfloat_mem_mask0(ptr %vp, <16 x float> %mask) {1071; CHECK-LABEL: test_masked_z_8xfloat_to_16xfloat_mem_mask0:1072; CHECK: # %bb.0:1073; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm11074; CHECK-NEXT: vcmpeqps %zmm1, %zmm0, %k11075; CHECK-NEXT: vbroadcastf32x8 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]1076; CHECK-NEXT: retq1077 %vec = load <8 x float>, ptr %vp1078 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>1079 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer1080 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer1081 ret <16 x float> %res1082}1083define <16 x float> @test_masked_8xfloat_to_16xfloat_mem_mask1(ptr %vp, <16 x float> %default, <16 x float> %mask) {1084; CHECK-LABEL: test_masked_8xfloat_to_16xfloat_mem_mask1:1085; CHECK: # %bb.0:1086; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm21087; CHECK-NEXT: vcmpeqps %zmm2, %zmm1, %k11088; CHECK-NEXT: vbroadcastf32x8 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]1089; CHECK-NEXT: retq1090 %vec = load <8 x float>, ptr %vp1091 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>1092 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer1093 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %default1094 ret <16 x float> %res1095}1096 1097define <16 x float> @test_masked_z_8xfloat_to_16xfloat_mem_mask1(ptr %vp, <16 x float> %mask) {1098; CHECK-LABEL: test_masked_z_8xfloat_to_16xfloat_mem_mask1:1099; CHECK: # %bb.0:1100; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm11101; CHECK-NEXT: vcmpeqps %zmm1, %zmm0, %k11102; CHECK-NEXT: vbroadcastf32x8 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]1103; CHECK-NEXT: retq1104 %vec = load <8 x float>, ptr %vp1105 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>1106 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer1107 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer1108 ret <16 x float> %res1109}1110define <16 x float> @test_masked_8xfloat_to_16xfloat_mem_mask2(ptr %vp, <16 x float> %default, <16 x float> %mask) {1111; CHECK-LABEL: test_masked_8xfloat_to_16xfloat_mem_mask2:1112; CHECK: # %bb.0:1113; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm21114; CHECK-NEXT: vcmpeqps %zmm2, %zmm1, %k11115; CHECK-NEXT: vbroadcastf32x8 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]1116; CHECK-NEXT: retq1117 %vec = load <8 x float>, ptr %vp1118 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>1119 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer1120 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %default1121 ret <16 x float> %res1122}1123 1124define <16 x float> @test_masked_z_8xfloat_to_16xfloat_mem_mask2(ptr %vp, <16 x float> %mask) {1125; CHECK-LABEL: test_masked_z_8xfloat_to_16xfloat_mem_mask2:1126; CHECK: # %bb.0:1127; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm11128; CHECK-NEXT: vcmpeqps %zmm1, %zmm0, %k11129; CHECK-NEXT: vbroadcastf32x8 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]1130; CHECK-NEXT: retq1131 %vec = load <8 x float>, ptr %vp1132 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>1133 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer1134 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer1135 ret <16 x float> %res1136}1137define <16 x float> @test_masked_8xfloat_to_16xfloat_mem_mask3(ptr %vp, <16 x float> %default, <16 x float> %mask) {1138; CHECK-LABEL: test_masked_8xfloat_to_16xfloat_mem_mask3:1139; CHECK: # %bb.0:1140; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm21141; CHECK-NEXT: vcmpeqps %zmm2, %zmm1, %k11142; CHECK-NEXT: vbroadcastf32x8 {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]1143; CHECK-NEXT: retq1144 %vec = load <8 x float>, ptr %vp1145 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>1146 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer1147 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %default1148 ret <16 x float> %res1149}1150 1151define <16 x float> @test_masked_z_8xfloat_to_16xfloat_mem_mask3(ptr %vp, <16 x float> %mask) {1152; CHECK-LABEL: test_masked_z_8xfloat_to_16xfloat_mem_mask3:1153; CHECK: # %bb.0:1154; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm11155; CHECK-NEXT: vcmpeqps %zmm1, %zmm0, %k11156; CHECK-NEXT: vbroadcastf32x8 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7]1157; CHECK-NEXT: retq1158 %vec = load <8 x float>, ptr %vp1159 %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>1160 %cmp = fcmp oeq <16 x float> %mask, zeroinitializer1161 %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer1162 ret <16 x float> %res1163}1164