1623 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle -O2 | FileCheck %s --check-prefixes=AVX512,AVX512-FAST3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl,+avx512dq,+fast-variable-perlane-shuffle -O2 | FileCheck %s --check-prefixes=AVX512,AVX512-FAST-PERLANE4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle -O2 | FileCheck %s --check-prefixes=AVX512NOTDQ,AVX512NOTDQ-FAST5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl,+fast-variable-perlane-shuffle -O2 | FileCheck %s --check-prefixes=AVX512NOTDQ,AVX512NOTDQ-FAST-PERLANE6 7define void @load_v8i1_broadcast_4_v2i1(ptr %a0,<2 x double> %a1,<2 x double> %a2,ptr %a3) {8; AVX512-LABEL: load_v8i1_broadcast_4_v2i1:9; AVX512: # %bb.0:10; AVX512-NEXT: kmovb (%rdi), %k011; AVX512-NEXT: kshiftrb $4, %k0, %k012; AVX512-NEXT: vpmovm2q %k0, %xmm213; AVX512-NEXT: vpbroadcastq %xmm2, %xmm214; AVX512-NEXT: vpmovq2m %xmm2, %k115; AVX512-NEXT: vmovapd %xmm0, %xmm1 {%k1}16; AVX512-NEXT: vmovapd %xmm1, (%rsi)17; AVX512-NEXT: retq18;19; AVX512NOTDQ-LABEL: load_v8i1_broadcast_4_v2i1:20; AVX512NOTDQ: # %bb.0:21; AVX512NOTDQ-NEXT: movzbl (%rdi), %eax22; AVX512NOTDQ-NEXT: kmovd %eax, %k023; AVX512NOTDQ-NEXT: kshiftrw $4, %k0, %k124; AVX512NOTDQ-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm225; AVX512NOTDQ-NEXT: vmovdqa64 %xmm2, %xmm2 {%k1} {z}26; AVX512NOTDQ-NEXT: vpbroadcastq %xmm2, %xmm227; AVX512NOTDQ-NEXT: vptestmq %xmm2, %xmm2, %k128; AVX512NOTDQ-NEXT: vmovapd %xmm0, %xmm1 {%k1}29; AVX512NOTDQ-NEXT: vmovapd %xmm1, (%rsi)30; AVX512NOTDQ-NEXT: retq31 %d0 = load <8 x i1>, ptr %a032 %d1 = shufflevector <8 x i1> %d0,<8 x i1> undef,<2 x i32><i32 4,i32 4>33 %d2 = select <2 x i1> %d1, <2 x double> %a1, <2 x double> %a234 store <2 x double> %d2, ptr %a335 ret void36}37define void @load_v8i1_broadcast_7_v2i1(ptr %a0,<2 x double> %a1,<2 x double> %a2,ptr %a3) {38; AVX512-LABEL: load_v8i1_broadcast_7_v2i1:39; AVX512: # %bb.0:40; AVX512-NEXT: kmovb (%rdi), %k041; AVX512-NEXT: kshiftrb $6, %k0, %k042; AVX512-NEXT: vpmovm2q %k0, %xmm243; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]44; AVX512-NEXT: vpmovq2m %xmm2, %k145; AVX512-NEXT: vmovapd %xmm0, %xmm1 {%k1}46; AVX512-NEXT: vmovapd %xmm1, (%rsi)47; AVX512-NEXT: retq48;49; AVX512NOTDQ-LABEL: load_v8i1_broadcast_7_v2i1:50; AVX512NOTDQ: # %bb.0:51; AVX512NOTDQ-NEXT: movzbl (%rdi), %eax52; AVX512NOTDQ-NEXT: kmovd %eax, %k053; AVX512NOTDQ-NEXT: kshiftrw $6, %k0, %k154; AVX512NOTDQ-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm255; AVX512NOTDQ-NEXT: vmovdqa64 %xmm2, %xmm2 {%k1} {z}56; AVX512NOTDQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]57; AVX512NOTDQ-NEXT: vptestmq %xmm2, %xmm2, %k158; AVX512NOTDQ-NEXT: vmovapd %xmm0, %xmm1 {%k1}59; AVX512NOTDQ-NEXT: vmovapd %xmm1, (%rsi)60; AVX512NOTDQ-NEXT: retq61 %d0 = load <8 x i1>, ptr %a062 %d1 = shufflevector <8 x i1> %d0,<8 x i1> undef,<2 x i32><i32 7,i32 7>63 %d2 = select <2 x i1> %d1, <2 x double> %a1, <2 x double> %a264 store <2 x double> %d2, ptr %a365 ret void66}67define void @load_v16i1_broadcast_8_v2i1(ptr %a0,<2 x double> %a1,<2 x double> %a2,ptr %a3) {68; AVX512-LABEL: load_v16i1_broadcast_8_v2i1:69; AVX512: # %bb.0:70; AVX512-NEXT: kmovw (%rdi), %k071; AVX512-NEXT: kshiftrw $8, %k0, %k072; AVX512-NEXT: vpmovm2q %k0, %xmm273; AVX512-NEXT: vpbroadcastq %xmm2, %xmm274; AVX512-NEXT: vpmovq2m %xmm2, %k175; AVX512-NEXT: vmovapd %xmm0, %xmm1 {%k1}76; AVX512-NEXT: vmovapd %xmm1, (%rsi)77; AVX512-NEXT: retq78;79; AVX512NOTDQ-LABEL: load_v16i1_broadcast_8_v2i1:80; AVX512NOTDQ: # %bb.0:81; AVX512NOTDQ-NEXT: kmovw (%rdi), %k082; AVX512NOTDQ-NEXT: kshiftrw $8, %k0, %k183; AVX512NOTDQ-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm284; AVX512NOTDQ-NEXT: vmovdqa64 %xmm2, %xmm2 {%k1} {z}85; AVX512NOTDQ-NEXT: vpbroadcastq %xmm2, %xmm286; AVX512NOTDQ-NEXT: vptestmq %xmm2, %xmm2, %k187; AVX512NOTDQ-NEXT: vmovapd %xmm0, %xmm1 {%k1}88; AVX512NOTDQ-NEXT: vmovapd %xmm1, (%rsi)89; AVX512NOTDQ-NEXT: retq90 %d0 = load <16 x i1>, ptr %a091 %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<2 x i32><i32 8,i32 8>92 %d2 = select <2 x i1> %d1, <2 x double> %a1, <2 x double> %a293 store <2 x double> %d2, ptr %a394 ret void95}96define void @load_v16i1_broadcast_8_v4i1(ptr %a0,<4 x float> %a1,<4 x float> %a2,ptr %a3) {97; AVX512-LABEL: load_v16i1_broadcast_8_v4i1:98; AVX512: # %bb.0:99; AVX512-NEXT: kmovw (%rdi), %k0100; AVX512-NEXT: kshiftrw $8, %k0, %k0101; AVX512-NEXT: vpmovm2d %k0, %xmm2102; AVX512-NEXT: vpbroadcastd %xmm2, %xmm2103; AVX512-NEXT: vpmovd2m %xmm2, %k1104; AVX512-NEXT: vmovaps %xmm0, %xmm1 {%k1}105; AVX512-NEXT: vmovaps %xmm1, (%rsi)106; AVX512-NEXT: retq107;108; AVX512NOTDQ-LABEL: load_v16i1_broadcast_8_v4i1:109; AVX512NOTDQ: # %bb.0:110; AVX512NOTDQ-NEXT: kmovw (%rdi), %k0111; AVX512NOTDQ-NEXT: kshiftrw $8, %k0, %k1112; AVX512NOTDQ-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2113; AVX512NOTDQ-NEXT: vmovdqa32 %xmm2, %xmm2 {%k1} {z}114; AVX512NOTDQ-NEXT: vpbroadcastd %xmm2, %xmm2115; AVX512NOTDQ-NEXT: vptestmd %xmm2, %xmm2, %k1116; AVX512NOTDQ-NEXT: vmovaps %xmm0, %xmm1 {%k1}117; AVX512NOTDQ-NEXT: vmovaps %xmm1, (%rsi)118; AVX512NOTDQ-NEXT: retq119 %d0 = load <16 x i1>, ptr %a0120 %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<4 x i32><i32 8,i32 8,i32 8,i32 8>121 %d2 = select <4 x i1> %d1, <4 x float> %a1, <4 x float> %a2122 store <4 x float> %d2, ptr %a3123 ret void124}125define void @load_v16i1_broadcast_15_v2i1(ptr %a0,<2 x double> %a1,<2 x double> %a2,ptr %a3) {126; AVX512-LABEL: load_v16i1_broadcast_15_v2i1:127; AVX512: # %bb.0:128; AVX512-NEXT: kmovw (%rdi), %k0129; AVX512-NEXT: kshiftrw $14, %k0, %k0130; AVX512-NEXT: vpmovm2q %k0, %xmm2131; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]132; AVX512-NEXT: vpmovq2m %xmm2, %k1133; AVX512-NEXT: vmovapd %xmm0, %xmm1 {%k1}134; AVX512-NEXT: vmovapd %xmm1, (%rsi)135; AVX512-NEXT: retq136;137; AVX512NOTDQ-LABEL: load_v16i1_broadcast_15_v2i1:138; AVX512NOTDQ: # %bb.0:139; AVX512NOTDQ-NEXT: kmovw (%rdi), %k0140; AVX512NOTDQ-NEXT: kshiftrw $14, %k0, %k1141; AVX512NOTDQ-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2142; AVX512NOTDQ-NEXT: vmovdqa64 %xmm2, %xmm2 {%k1} {z}143; AVX512NOTDQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]144; AVX512NOTDQ-NEXT: vptestmq %xmm2, %xmm2, %k1145; AVX512NOTDQ-NEXT: vmovapd %xmm0, %xmm1 {%k1}146; AVX512NOTDQ-NEXT: vmovapd %xmm1, (%rsi)147; AVX512NOTDQ-NEXT: retq148 %d0 = load <16 x i1>, ptr %a0149 %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<2 x i32><i32 15,i32 15>150 %d2 = select <2 x i1> %d1, <2 x double> %a1, <2 x double> %a2151 store <2 x double> %d2, ptr %a3152 ret void153}154define void @load_v16i1_broadcast_15_v4i1(ptr %a0,<4 x float> %a1,<4 x float> %a2,ptr %a3) {155; AVX512-LABEL: load_v16i1_broadcast_15_v4i1:156; AVX512: # %bb.0:157; AVX512-NEXT: kmovw (%rdi), %k0158; AVX512-NEXT: kshiftrw $12, %k0, %k0159; AVX512-NEXT: vpmovm2d %k0, %xmm2160; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]161; AVX512-NEXT: vpmovd2m %xmm2, %k1162; AVX512-NEXT: vmovaps %xmm0, %xmm1 {%k1}163; AVX512-NEXT: vmovaps %xmm1, (%rsi)164; AVX512-NEXT: retq165;166; AVX512NOTDQ-LABEL: load_v16i1_broadcast_15_v4i1:167; AVX512NOTDQ: # %bb.0:168; AVX512NOTDQ-NEXT: kmovw (%rdi), %k0169; AVX512NOTDQ-NEXT: kshiftrw $12, %k0, %k1170; AVX512NOTDQ-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2171; AVX512NOTDQ-NEXT: vmovdqa32 %xmm2, %xmm2 {%k1} {z}172; AVX512NOTDQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]173; AVX512NOTDQ-NEXT: vptestmd %xmm2, %xmm2, %k1174; AVX512NOTDQ-NEXT: vmovaps %xmm0, %xmm1 {%k1}175; AVX512NOTDQ-NEXT: vmovaps %xmm1, (%rsi)176; AVX512NOTDQ-NEXT: retq177 %d0 = load <16 x i1>, ptr %a0178 %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<4 x i32><i32 15,i32 15,i32 15,i32 15>179 %d2 = select <4 x i1> %d1, <4 x float> %a1, <4 x float> %a2180 store <4 x float> %d2, ptr %a3181 ret void182}183define void @load_v32i1_broadcast_16_v2i1(ptr %a0,<2 x double> %a1,<2 x double> %a2,ptr %a3) {184; AVX512-LABEL: load_v32i1_broadcast_16_v2i1:185; AVX512: # %bb.0:186; AVX512-NEXT: kmovd (%rdi), %k0187; AVX512-NEXT: kshiftrd $16, %k0, %k0188; AVX512-NEXT: vpmovm2q %k0, %xmm2189; AVX512-NEXT: vpbroadcastq %xmm2, %xmm2190; AVX512-NEXT: vpmovq2m %xmm2, %k1191; AVX512-NEXT: vmovapd %xmm0, %xmm1 {%k1}192; AVX512-NEXT: vmovapd %xmm1, (%rsi)193; AVX512-NEXT: retq194;195; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v2i1:196; AVX512NOTDQ: # %bb.0:197; AVX512NOTDQ-NEXT: kmovd (%rdi), %k0198; AVX512NOTDQ-NEXT: kshiftrd $16, %k0, %k1199; AVX512NOTDQ-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2200; AVX512NOTDQ-NEXT: vmovdqa64 %xmm2, %xmm2 {%k1} {z}201; AVX512NOTDQ-NEXT: vpbroadcastq %xmm2, %xmm2202; AVX512NOTDQ-NEXT: vptestmq %xmm2, %xmm2, %k1203; AVX512NOTDQ-NEXT: vmovapd %xmm0, %xmm1 {%k1}204; AVX512NOTDQ-NEXT: vmovapd %xmm1, (%rsi)205; AVX512NOTDQ-NEXT: retq206 %d0 = load <32 x i1>, ptr %a0207 %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<2 x i32><i32 16,i32 16>208 %d2 = select <2 x i1> %d1, <2 x double> %a1, <2 x double> %a2209 store <2 x double> %d2, ptr %a3210 ret void211}212define void @load_v32i1_broadcast_16_v4i1(ptr %a0,<4 x float> %a1,<4 x float> %a2,ptr %a3) {213; AVX512-LABEL: load_v32i1_broadcast_16_v4i1:214; AVX512: # %bb.0:215; AVX512-NEXT: kmovd (%rdi), %k0216; AVX512-NEXT: kshiftrd $16, %k0, %k0217; AVX512-NEXT: vpmovm2d %k0, %xmm2218; AVX512-NEXT: vpbroadcastd %xmm2, %xmm2219; AVX512-NEXT: vpmovd2m %xmm2, %k1220; AVX512-NEXT: vmovaps %xmm0, %xmm1 {%k1}221; AVX512-NEXT: vmovaps %xmm1, (%rsi)222; AVX512-NEXT: retq223;224; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v4i1:225; AVX512NOTDQ: # %bb.0:226; AVX512NOTDQ-NEXT: kmovd (%rdi), %k0227; AVX512NOTDQ-NEXT: kshiftrd $16, %k0, %k1228; AVX512NOTDQ-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2229; AVX512NOTDQ-NEXT: vmovdqa32 %xmm2, %xmm2 {%k1} {z}230; AVX512NOTDQ-NEXT: vpbroadcastd %xmm2, %xmm2231; AVX512NOTDQ-NEXT: vptestmd %xmm2, %xmm2, %k1232; AVX512NOTDQ-NEXT: vmovaps %xmm0, %xmm1 {%k1}233; AVX512NOTDQ-NEXT: vmovaps %xmm1, (%rsi)234; AVX512NOTDQ-NEXT: retq235 %d0 = load <32 x i1>, ptr %a0236 %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<4 x i32><i32 16,i32 16,i32 16,i32 16>237 %d2 = select <4 x i1> %d1, <4 x float> %a1, <4 x float> %a2238 store <4 x float> %d2, ptr %a3239 ret void240}241define void @load_v32i1_broadcast_16_v8i1(ptr %a0,<8 x float> %a1,<8 x float> %a2,ptr %a3) {242; AVX512-LABEL: load_v32i1_broadcast_16_v8i1:243; AVX512: # %bb.0:244; AVX512-NEXT: kmovb 2(%rdi), %k0245; AVX512-NEXT: vpmovm2d %k0, %ymm2246; AVX512-NEXT: vpbroadcastd %xmm2, %ymm2247; AVX512-NEXT: vpmovd2m %ymm2, %k1248; AVX512-NEXT: vmovaps %ymm0, %ymm1 {%k1}249; AVX512-NEXT: vmovaps %ymm1, (%rsi)250; AVX512-NEXT: vzeroupper251; AVX512-NEXT: retq252;253; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v8i1:254; AVX512NOTDQ: # %bb.0:255; AVX512NOTDQ-NEXT: kmovw 2(%rdi), %k1256; AVX512NOTDQ-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2257; AVX512NOTDQ-NEXT: vmovdqa32 %ymm2, %ymm2 {%k1} {z}258; AVX512NOTDQ-NEXT: vpbroadcastd %xmm2, %ymm2259; AVX512NOTDQ-NEXT: vptestmd %ymm2, %ymm2, %k1260; AVX512NOTDQ-NEXT: vmovaps %ymm0, %ymm1 {%k1}261; AVX512NOTDQ-NEXT: vmovaps %ymm1, (%rsi)262; AVX512NOTDQ-NEXT: vzeroupper263; AVX512NOTDQ-NEXT: retq264 %d0 = load <32 x i1>, ptr %a0265 %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<8 x i32><i32 16,i32 16,i32 16,i32 16,i32 16,i32 16,i32 16,i32 16>266 %d2 = select <8 x i1> %d1, <8 x float> %a1, <8 x float> %a2267 store <8 x float> %d2, ptr %a3268 ret void269}270define void @load_v32i1_broadcast_31_v2i1(ptr %a0,<2 x double> %a1,<2 x double> %a2,ptr %a3) {271; AVX512-LABEL: load_v32i1_broadcast_31_v2i1:272; AVX512: # %bb.0:273; AVX512-NEXT: kmovd (%rdi), %k0274; AVX512-NEXT: kshiftrd $30, %k0, %k0275; AVX512-NEXT: vpmovm2q %k0, %xmm2276; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]277; AVX512-NEXT: vpmovq2m %xmm2, %k1278; AVX512-NEXT: vmovapd %xmm0, %xmm1 {%k1}279; AVX512-NEXT: vmovapd %xmm1, (%rsi)280; AVX512-NEXT: retq281;282; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v2i1:283; AVX512NOTDQ: # %bb.0:284; AVX512NOTDQ-NEXT: kmovd (%rdi), %k0285; AVX512NOTDQ-NEXT: kshiftrd $30, %k0, %k1286; AVX512NOTDQ-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2287; AVX512NOTDQ-NEXT: vmovdqa64 %xmm2, %xmm2 {%k1} {z}288; AVX512NOTDQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]289; AVX512NOTDQ-NEXT: vptestmq %xmm2, %xmm2, %k1290; AVX512NOTDQ-NEXT: vmovapd %xmm0, %xmm1 {%k1}291; AVX512NOTDQ-NEXT: vmovapd %xmm1, (%rsi)292; AVX512NOTDQ-NEXT: retq293 %d0 = load <32 x i1>, ptr %a0294 %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<2 x i32><i32 31,i32 31>295 %d2 = select <2 x i1> %d1, <2 x double> %a1, <2 x double> %a2296 store <2 x double> %d2, ptr %a3297 ret void298}299define void @load_v32i1_broadcast_31_v4i1(ptr %a0,<4 x float> %a1,<4 x float> %a2,ptr %a3) {300; AVX512-LABEL: load_v32i1_broadcast_31_v4i1:301; AVX512: # %bb.0:302; AVX512-NEXT: kmovd (%rdi), %k0303; AVX512-NEXT: kshiftrd $28, %k0, %k0304; AVX512-NEXT: vpmovm2d %k0, %xmm2305; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]306; AVX512-NEXT: vpmovd2m %xmm2, %k1307; AVX512-NEXT: vmovaps %xmm0, %xmm1 {%k1}308; AVX512-NEXT: vmovaps %xmm1, (%rsi)309; AVX512-NEXT: retq310;311; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v4i1:312; AVX512NOTDQ: # %bb.0:313; AVX512NOTDQ-NEXT: kmovd (%rdi), %k0314; AVX512NOTDQ-NEXT: kshiftrd $28, %k0, %k1315; AVX512NOTDQ-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2316; AVX512NOTDQ-NEXT: vmovdqa32 %xmm2, %xmm2 {%k1} {z}317; AVX512NOTDQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]318; AVX512NOTDQ-NEXT: vptestmd %xmm2, %xmm2, %k1319; AVX512NOTDQ-NEXT: vmovaps %xmm0, %xmm1 {%k1}320; AVX512NOTDQ-NEXT: vmovaps %xmm1, (%rsi)321; AVX512NOTDQ-NEXT: retq322 %d0 = load <32 x i1>, ptr %a0323 %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<4 x i32><i32 31,i32 31,i32 31,i32 31>324 %d2 = select <4 x i1> %d1, <4 x float> %a1, <4 x float> %a2325 store <4 x float> %d2, ptr %a3326 ret void327}328define void @load_v32i1_broadcast_31_v8i1(ptr %a0,<8 x float> %a1,<8 x float> %a2,ptr %a3) {329; AVX512-FAST-LABEL: load_v32i1_broadcast_31_v8i1:330; AVX512-FAST: # %bb.0:331; AVX512-FAST-NEXT: kmovb 3(%rdi), %k0332; AVX512-FAST-NEXT: vpmovm2d %k0, %ymm2333; AVX512-FAST-NEXT: vpbroadcastd {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7]334; AVX512-FAST-NEXT: vpermd %ymm2, %ymm3, %ymm2335; AVX512-FAST-NEXT: vpmovd2m %ymm2, %k1336; AVX512-FAST-NEXT: vmovaps %ymm0, %ymm1 {%k1}337; AVX512-FAST-NEXT: vmovaps %ymm1, (%rsi)338; AVX512-FAST-NEXT: vzeroupper339; AVX512-FAST-NEXT: retq340;341; AVX512-FAST-PERLANE-LABEL: load_v32i1_broadcast_31_v8i1:342; AVX512-FAST-PERLANE: # %bb.0:343; AVX512-FAST-PERLANE-NEXT: kmovb 3(%rdi), %k0344; AVX512-FAST-PERLANE-NEXT: vpmovm2d %k0, %ymm2345; AVX512-FAST-PERLANE-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[3,3,3,3,7,7,7,7]346; AVX512-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]347; AVX512-FAST-PERLANE-NEXT: vpmovd2m %ymm2, %k1348; AVX512-FAST-PERLANE-NEXT: vmovaps %ymm0, %ymm1 {%k1}349; AVX512-FAST-PERLANE-NEXT: vmovaps %ymm1, (%rsi)350; AVX512-FAST-PERLANE-NEXT: vzeroupper351; AVX512-FAST-PERLANE-NEXT: retq352;353; AVX512NOTDQ-FAST-LABEL: load_v32i1_broadcast_31_v8i1:354; AVX512NOTDQ-FAST: # %bb.0:355; AVX512NOTDQ-FAST-NEXT: movzbl 3(%rdi), %eax356; AVX512NOTDQ-FAST-NEXT: kmovd %eax, %k1357; AVX512NOTDQ-FAST-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2358; AVX512NOTDQ-FAST-NEXT: vmovdqa32 %ymm2, %ymm2 {%k1} {z}359; AVX512NOTDQ-FAST-NEXT: vpbroadcastd {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7]360; AVX512NOTDQ-FAST-NEXT: vpermd %ymm2, %ymm3, %ymm2361; AVX512NOTDQ-FAST-NEXT: vptestmd %ymm2, %ymm2, %k1362; AVX512NOTDQ-FAST-NEXT: vmovaps %ymm0, %ymm1 {%k1}363; AVX512NOTDQ-FAST-NEXT: vmovaps %ymm1, (%rsi)364; AVX512NOTDQ-FAST-NEXT: vzeroupper365; AVX512NOTDQ-FAST-NEXT: retq366;367; AVX512NOTDQ-FAST-PERLANE-LABEL: load_v32i1_broadcast_31_v8i1:368; AVX512NOTDQ-FAST-PERLANE: # %bb.0:369; AVX512NOTDQ-FAST-PERLANE-NEXT: movzbl 3(%rdi), %eax370; AVX512NOTDQ-FAST-PERLANE-NEXT: kmovd %eax, %k1371; AVX512NOTDQ-FAST-PERLANE-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2372; AVX512NOTDQ-FAST-PERLANE-NEXT: vmovdqa32 %ymm2, %ymm2 {%k1} {z}373; AVX512NOTDQ-FAST-PERLANE-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[3,3,3,3,7,7,7,7]374; AVX512NOTDQ-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]375; AVX512NOTDQ-FAST-PERLANE-NEXT: vptestmd %ymm2, %ymm2, %k1376; AVX512NOTDQ-FAST-PERLANE-NEXT: vmovaps %ymm0, %ymm1 {%k1}377; AVX512NOTDQ-FAST-PERLANE-NEXT: vmovaps %ymm1, (%rsi)378; AVX512NOTDQ-FAST-PERLANE-NEXT: vzeroupper379; AVX512NOTDQ-FAST-PERLANE-NEXT: retq380 %d0 = load <32 x i1>, ptr %a0381 %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<8 x i32><i32 31,i32 31,i32 31,i32 31,i32 31,i32 31,i32 31,i32 31>382 %d2 = select <8 x i1> %d1, <8 x float> %a1, <8 x float> %a2383 store <8 x float> %d2, ptr %a3384 ret void385}386define void @load_v64i1_broadcast_32_v2i1(ptr %a0,<2 x double> %a1,<2 x double> %a2,ptr %a3) {387; AVX512-LABEL: load_v64i1_broadcast_32_v2i1:388; AVX512: # %bb.0:389; AVX512-NEXT: kmovq (%rdi), %k0390; AVX512-NEXT: kshiftrq $32, %k0, %k0391; AVX512-NEXT: vpmovm2q %k0, %xmm2392; AVX512-NEXT: vpbroadcastq %xmm2, %xmm2393; AVX512-NEXT: vpmovq2m %xmm2, %k1394; AVX512-NEXT: vmovapd %xmm0, %xmm1 {%k1}395; AVX512-NEXT: vmovapd %xmm1, (%rsi)396; AVX512-NEXT: retq397;398; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v2i1:399; AVX512NOTDQ: # %bb.0:400; AVX512NOTDQ-NEXT: kmovq (%rdi), %k0401; AVX512NOTDQ-NEXT: kshiftrq $32, %k0, %k1402; AVX512NOTDQ-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2403; AVX512NOTDQ-NEXT: vmovdqa64 %xmm2, %xmm2 {%k1} {z}404; AVX512NOTDQ-NEXT: vpbroadcastq %xmm2, %xmm2405; AVX512NOTDQ-NEXT: vptestmq %xmm2, %xmm2, %k1406; AVX512NOTDQ-NEXT: vmovapd %xmm0, %xmm1 {%k1}407; AVX512NOTDQ-NEXT: vmovapd %xmm1, (%rsi)408; AVX512NOTDQ-NEXT: retq409 %d0 = load <64 x i1>, ptr %a0410 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<2 x i32><i32 32,i32 32>411 %d2 = select <2 x i1> %d1, <2 x double> %a1, <2 x double> %a2412 store <2 x double> %d2, ptr %a3413 ret void414}415define void @load_v64i1_broadcast_32_v4i1(ptr %a0,<4 x float> %a1,<4 x float> %a2,ptr %a3) {416; AVX512-LABEL: load_v64i1_broadcast_32_v4i1:417; AVX512: # %bb.0:418; AVX512-NEXT: kmovq (%rdi), %k0419; AVX512-NEXT: kshiftrq $32, %k0, %k0420; AVX512-NEXT: vpmovm2d %k0, %xmm2421; AVX512-NEXT: vpbroadcastd %xmm2, %xmm2422; AVX512-NEXT: vpmovd2m %xmm2, %k1423; AVX512-NEXT: vmovaps %xmm0, %xmm1 {%k1}424; AVX512-NEXT: vmovaps %xmm1, (%rsi)425; AVX512-NEXT: retq426;427; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v4i1:428; AVX512NOTDQ: # %bb.0:429; AVX512NOTDQ-NEXT: kmovq (%rdi), %k0430; AVX512NOTDQ-NEXT: kshiftrq $32, %k0, %k1431; AVX512NOTDQ-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2432; AVX512NOTDQ-NEXT: vmovdqa32 %xmm2, %xmm2 {%k1} {z}433; AVX512NOTDQ-NEXT: vpbroadcastd %xmm2, %xmm2434; AVX512NOTDQ-NEXT: vptestmd %xmm2, %xmm2, %k1435; AVX512NOTDQ-NEXT: vmovaps %xmm0, %xmm1 {%k1}436; AVX512NOTDQ-NEXT: vmovaps %xmm1, (%rsi)437; AVX512NOTDQ-NEXT: retq438 %d0 = load <64 x i1>, ptr %a0439 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<4 x i32><i32 32,i32 32,i32 32,i32 32>440 %d2 = select <4 x i1> %d1, <4 x float> %a1, <4 x float> %a2441 store <4 x float> %d2, ptr %a3442 ret void443}444define void @load_v64i1_broadcast_32_v8i1(ptr %a0,<8 x float> %a1,<8 x float> %a2,ptr %a3) {445; AVX512-LABEL: load_v64i1_broadcast_32_v8i1:446; AVX512: # %bb.0:447; AVX512-NEXT: kmovb 4(%rdi), %k0448; AVX512-NEXT: vpmovm2d %k0, %ymm2449; AVX512-NEXT: vpbroadcastd %xmm2, %ymm2450; AVX512-NEXT: vpmovd2m %ymm2, %k1451; AVX512-NEXT: vmovaps %ymm0, %ymm1 {%k1}452; AVX512-NEXT: vmovaps %ymm1, (%rsi)453; AVX512-NEXT: vzeroupper454; AVX512-NEXT: retq455;456; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v8i1:457; AVX512NOTDQ: # %bb.0:458; AVX512NOTDQ-NEXT: kmovw 4(%rdi), %k1459; AVX512NOTDQ-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2460; AVX512NOTDQ-NEXT: vmovdqa32 %ymm2, %ymm2 {%k1} {z}461; AVX512NOTDQ-NEXT: vpbroadcastd %xmm2, %ymm2462; AVX512NOTDQ-NEXT: vptestmd %ymm2, %ymm2, %k1463; AVX512NOTDQ-NEXT: vmovaps %ymm0, %ymm1 {%k1}464; AVX512NOTDQ-NEXT: vmovaps %ymm1, (%rsi)465; AVX512NOTDQ-NEXT: vzeroupper466; AVX512NOTDQ-NEXT: retq467 %d0 = load <64 x i1>, ptr %a0468 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<8 x i32><i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32>469 %d2 = select <8 x i1> %d1, <8 x float> %a1, <8 x float> %a2470 store <8 x float> %d2, ptr %a3471 ret void472}473define void @load_v64i1_broadcast_32_v16i1(ptr %a0,<16 x float> %a1,<16 x float> %a2,ptr %a3) {474; AVX512-LABEL: load_v64i1_broadcast_32_v16i1:475; AVX512: # %bb.0:476; AVX512-NEXT: kmovw 4(%rdi), %k0477; AVX512-NEXT: vpmovm2d %k0, %zmm2478; AVX512-NEXT: vpbroadcastd %xmm2, %zmm2479; AVX512-NEXT: vpmovd2m %zmm2, %k1480; AVX512-NEXT: vmovaps %zmm0, %zmm1 {%k1}481; AVX512-NEXT: vmovaps %zmm1, (%rsi)482; AVX512-NEXT: vzeroupper483; AVX512-NEXT: retq484;485; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v16i1:486; AVX512NOTDQ: # %bb.0:487; AVX512NOTDQ-NEXT: kmovw 4(%rdi), %k1488; AVX512NOTDQ-NEXT: vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1489; AVX512NOTDQ-NEXT: vpbroadcastd %xmm2, %zmm2490; AVX512NOTDQ-NEXT: vptestmd %zmm2, %zmm2, %k1491; AVX512NOTDQ-NEXT: vmovaps %zmm0, %zmm1 {%k1}492; AVX512NOTDQ-NEXT: vmovaps %zmm1, (%rsi)493; AVX512NOTDQ-NEXT: vzeroupper494; AVX512NOTDQ-NEXT: retq495 %d0 = load <64 x i1>, ptr %a0496 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<16 x i32><i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32>497 %d2 = select <16 x i1> %d1, <16 x float> %a1, <16 x float> %a2498 store <16 x float> %d2, ptr %a3499 ret void500}501define void @load_v64i1_broadcast_63_v2i1(ptr %a0,<2 x double> %a1,<2 x double> %a2,ptr %a3) {502; AVX512-LABEL: load_v64i1_broadcast_63_v2i1:503; AVX512: # %bb.0:504; AVX512-NEXT: kmovq (%rdi), %k0505; AVX512-NEXT: kshiftrq $62, %k0, %k0506; AVX512-NEXT: vpmovm2q %k0, %xmm2507; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]508; AVX512-NEXT: vpmovq2m %xmm2, %k1509; AVX512-NEXT: vmovapd %xmm0, %xmm1 {%k1}510; AVX512-NEXT: vmovapd %xmm1, (%rsi)511; AVX512-NEXT: retq512;513; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v2i1:514; AVX512NOTDQ: # %bb.0:515; AVX512NOTDQ-NEXT: kmovq (%rdi), %k0516; AVX512NOTDQ-NEXT: kshiftrq $62, %k0, %k1517; AVX512NOTDQ-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2518; AVX512NOTDQ-NEXT: vmovdqa64 %xmm2, %xmm2 {%k1} {z}519; AVX512NOTDQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]520; AVX512NOTDQ-NEXT: vptestmq %xmm2, %xmm2, %k1521; AVX512NOTDQ-NEXT: vmovapd %xmm0, %xmm1 {%k1}522; AVX512NOTDQ-NEXT: vmovapd %xmm1, (%rsi)523; AVX512NOTDQ-NEXT: retq524 %d0 = load <64 x i1>, ptr %a0525 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<2 x i32><i32 63,i32 63>526 %d2 = select <2 x i1> %d1, <2 x double> %a1, <2 x double> %a2527 store <2 x double> %d2, ptr %a3528 ret void529}530define void @load_v64i1_broadcast_63_v4i1(ptr %a0,<4 x float> %a1,<4 x float> %a2,ptr %a3) {531; AVX512-LABEL: load_v64i1_broadcast_63_v4i1:532; AVX512: # %bb.0:533; AVX512-NEXT: kmovq (%rdi), %k0534; AVX512-NEXT: kshiftrq $60, %k0, %k0535; AVX512-NEXT: vpmovm2d %k0, %xmm2536; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]537; AVX512-NEXT: vpmovd2m %xmm2, %k1538; AVX512-NEXT: vmovaps %xmm0, %xmm1 {%k1}539; AVX512-NEXT: vmovaps %xmm1, (%rsi)540; AVX512-NEXT: retq541;542; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v4i1:543; AVX512NOTDQ: # %bb.0:544; AVX512NOTDQ-NEXT: kmovq (%rdi), %k0545; AVX512NOTDQ-NEXT: kshiftrq $60, %k0, %k1546; AVX512NOTDQ-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2547; AVX512NOTDQ-NEXT: vmovdqa32 %xmm2, %xmm2 {%k1} {z}548; AVX512NOTDQ-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]549; AVX512NOTDQ-NEXT: vptestmd %xmm2, %xmm2, %k1550; AVX512NOTDQ-NEXT: vmovaps %xmm0, %xmm1 {%k1}551; AVX512NOTDQ-NEXT: vmovaps %xmm1, (%rsi)552; AVX512NOTDQ-NEXT: retq553 %d0 = load <64 x i1>, ptr %a0554 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<4 x i32><i32 63,i32 63,i32 63,i32 63>555 %d2 = select <4 x i1> %d1, <4 x float> %a1, <4 x float> %a2556 store <4 x float> %d2, ptr %a3557 ret void558}559define void @load_v64i1_broadcast_63_v8i1(ptr %a0,<8 x float> %a1,<8 x float> %a2,ptr %a3) {560; AVX512-FAST-LABEL: load_v64i1_broadcast_63_v8i1:561; AVX512-FAST: # %bb.0:562; AVX512-FAST-NEXT: kmovb 7(%rdi), %k0563; AVX512-FAST-NEXT: vpmovm2d %k0, %ymm2564; AVX512-FAST-NEXT: vpbroadcastd {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7]565; AVX512-FAST-NEXT: vpermd %ymm2, %ymm3, %ymm2566; AVX512-FAST-NEXT: vpmovd2m %ymm2, %k1567; AVX512-FAST-NEXT: vmovaps %ymm0, %ymm1 {%k1}568; AVX512-FAST-NEXT: vmovaps %ymm1, (%rsi)569; AVX512-FAST-NEXT: vzeroupper570; AVX512-FAST-NEXT: retq571;572; AVX512-FAST-PERLANE-LABEL: load_v64i1_broadcast_63_v8i1:573; AVX512-FAST-PERLANE: # %bb.0:574; AVX512-FAST-PERLANE-NEXT: kmovb 7(%rdi), %k0575; AVX512-FAST-PERLANE-NEXT: vpmovm2d %k0, %ymm2576; AVX512-FAST-PERLANE-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[3,3,3,3,7,7,7,7]577; AVX512-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]578; AVX512-FAST-PERLANE-NEXT: vpmovd2m %ymm2, %k1579; AVX512-FAST-PERLANE-NEXT: vmovaps %ymm0, %ymm1 {%k1}580; AVX512-FAST-PERLANE-NEXT: vmovaps %ymm1, (%rsi)581; AVX512-FAST-PERLANE-NEXT: vzeroupper582; AVX512-FAST-PERLANE-NEXT: retq583;584; AVX512NOTDQ-FAST-LABEL: load_v64i1_broadcast_63_v8i1:585; AVX512NOTDQ-FAST: # %bb.0:586; AVX512NOTDQ-FAST-NEXT: movzbl 7(%rdi), %eax587; AVX512NOTDQ-FAST-NEXT: kmovd %eax, %k1588; AVX512NOTDQ-FAST-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2589; AVX512NOTDQ-FAST-NEXT: vmovdqa32 %ymm2, %ymm2 {%k1} {z}590; AVX512NOTDQ-FAST-NEXT: vpbroadcastd {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7]591; AVX512NOTDQ-FAST-NEXT: vpermd %ymm2, %ymm3, %ymm2592; AVX512NOTDQ-FAST-NEXT: vptestmd %ymm2, %ymm2, %k1593; AVX512NOTDQ-FAST-NEXT: vmovaps %ymm0, %ymm1 {%k1}594; AVX512NOTDQ-FAST-NEXT: vmovaps %ymm1, (%rsi)595; AVX512NOTDQ-FAST-NEXT: vzeroupper596; AVX512NOTDQ-FAST-NEXT: retq597;598; AVX512NOTDQ-FAST-PERLANE-LABEL: load_v64i1_broadcast_63_v8i1:599; AVX512NOTDQ-FAST-PERLANE: # %bb.0:600; AVX512NOTDQ-FAST-PERLANE-NEXT: movzbl 7(%rdi), %eax601; AVX512NOTDQ-FAST-PERLANE-NEXT: kmovd %eax, %k1602; AVX512NOTDQ-FAST-PERLANE-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2603; AVX512NOTDQ-FAST-PERLANE-NEXT: vmovdqa32 %ymm2, %ymm2 {%k1} {z}604; AVX512NOTDQ-FAST-PERLANE-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[3,3,3,3,7,7,7,7]605; AVX512NOTDQ-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,2,2]606; AVX512NOTDQ-FAST-PERLANE-NEXT: vptestmd %ymm2, %ymm2, %k1607; AVX512NOTDQ-FAST-PERLANE-NEXT: vmovaps %ymm0, %ymm1 {%k1}608; AVX512NOTDQ-FAST-PERLANE-NEXT: vmovaps %ymm1, (%rsi)609; AVX512NOTDQ-FAST-PERLANE-NEXT: vzeroupper610; AVX512NOTDQ-FAST-PERLANE-NEXT: retq611 %d0 = load <64 x i1>, ptr %a0612 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<8 x i32><i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63>613 %d2 = select <8 x i1> %d1, <8 x float> %a1, <8 x float> %a2614 store <8 x float> %d2, ptr %a3615 ret void616}617define void @load_v64i1_broadcast_63_v16i1(ptr %a0,<16 x float> %a1,<16 x float> %a2,ptr %a3) {618; AVX512-FAST-LABEL: load_v64i1_broadcast_63_v16i1:619; AVX512-FAST: # %bb.0:620; AVX512-FAST-NEXT: kmovw 6(%rdi), %k0621; AVX512-FAST-NEXT: vpmovm2d %k0, %zmm2622; AVX512-FAST-NEXT: vpbroadcastd {{.*#+}} zmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]623; AVX512-FAST-NEXT: vpermd %zmm2, %zmm3, %zmm2624; AVX512-FAST-NEXT: vpmovd2m %zmm2, %k1625; AVX512-FAST-NEXT: vmovaps %zmm0, %zmm1 {%k1}626; AVX512-FAST-NEXT: vmovaps %zmm1, (%rsi)627; AVX512-FAST-NEXT: vzeroupper628; AVX512-FAST-NEXT: retq629;630; AVX512-FAST-PERLANE-LABEL: load_v64i1_broadcast_63_v16i1:631; AVX512-FAST-PERLANE: # %bb.0:632; AVX512-FAST-PERLANE-NEXT: kmovw 6(%rdi), %k0633; AVX512-FAST-PERLANE-NEXT: vpmovm2d %k0, %zmm2634; AVX512-FAST-PERLANE-NEXT: vpshufd {{.*#+}} zmm2 = zmm2[3,3,3,3,7,7,7,7,11,11,11,11,15,15,15,15]635; AVX512-FAST-PERLANE-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[6,7,6,7,6,7,6,7]636; AVX512-FAST-PERLANE-NEXT: vpmovd2m %zmm2, %k1637; AVX512-FAST-PERLANE-NEXT: vmovaps %zmm0, %zmm1 {%k1}638; AVX512-FAST-PERLANE-NEXT: vmovaps %zmm1, (%rsi)639; AVX512-FAST-PERLANE-NEXT: vzeroupper640; AVX512-FAST-PERLANE-NEXT: retq641;642; AVX512NOTDQ-FAST-LABEL: load_v64i1_broadcast_63_v16i1:643; AVX512NOTDQ-FAST: # %bb.0:644; AVX512NOTDQ-FAST-NEXT: kmovw 6(%rdi), %k1645; AVX512NOTDQ-FAST-NEXT: vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1646; AVX512NOTDQ-FAST-NEXT: vpbroadcastd {{.*#+}} zmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]647; AVX512NOTDQ-FAST-NEXT: vpermd %zmm2, %zmm3, %zmm2648; AVX512NOTDQ-FAST-NEXT: vptestmd %zmm2, %zmm2, %k1649; AVX512NOTDQ-FAST-NEXT: vmovaps %zmm0, %zmm1 {%k1}650; AVX512NOTDQ-FAST-NEXT: vmovaps %zmm1, (%rsi)651; AVX512NOTDQ-FAST-NEXT: vzeroupper652; AVX512NOTDQ-FAST-NEXT: retq653;654; AVX512NOTDQ-FAST-PERLANE-LABEL: load_v64i1_broadcast_63_v16i1:655; AVX512NOTDQ-FAST-PERLANE: # %bb.0:656; AVX512NOTDQ-FAST-PERLANE-NEXT: kmovw 6(%rdi), %k1657; AVX512NOTDQ-FAST-PERLANE-NEXT: vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1658; AVX512NOTDQ-FAST-PERLANE-NEXT: vpshufd {{.*#+}} zmm2 = zmm2[3,3,3,3,7,7,7,7,11,11,11,11,15,15,15,15]659; AVX512NOTDQ-FAST-PERLANE-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[6,7,6,7,6,7,6,7]660; AVX512NOTDQ-FAST-PERLANE-NEXT: vptestmd %zmm2, %zmm2, %k1661; AVX512NOTDQ-FAST-PERLANE-NEXT: vmovaps %zmm0, %zmm1 {%k1}662; AVX512NOTDQ-FAST-PERLANE-NEXT: vmovaps %zmm1, (%rsi)663; AVX512NOTDQ-FAST-PERLANE-NEXT: vzeroupper664; AVX512NOTDQ-FAST-PERLANE-NEXT: retq665 %d0 = load <64 x i1>, ptr %a0666 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<16 x i32><i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63>667 %d2 = select <16 x i1> %d1, <16 x float> %a1, <16 x float> %a2668 store <16 x float> %d2, ptr %a3669 ret void670}671define void @load_v2i1_broadcast_1_v1i1_store(ptr %a0,ptr %a1) {672; AVX512-LABEL: load_v2i1_broadcast_1_v1i1_store:673; AVX512: # %bb.0:674; AVX512-NEXT: kmovb (%rdi), %k0675; AVX512-NEXT: kshiftrb $1, %k0, %k0676; AVX512-NEXT: kshiftlb $7, %k0, %k0677; AVX512-NEXT: kshiftrb $7, %k0, %k0678; AVX512-NEXT: kmovb %k0, (%rsi)679; AVX512-NEXT: retq680;681; AVX512NOTDQ-LABEL: load_v2i1_broadcast_1_v1i1_store:682; AVX512NOTDQ: # %bb.0:683; AVX512NOTDQ-NEXT: movzbl (%rdi), %eax684; AVX512NOTDQ-NEXT: kmovd %eax, %k0685; AVX512NOTDQ-NEXT: kshiftrw $1, %k0, %k0686; AVX512NOTDQ-NEXT: kshiftlw $15, %k0, %k0687; AVX512NOTDQ-NEXT: kshiftrw $15, %k0, %k0688; AVX512NOTDQ-NEXT: kmovd %k0, %eax689; AVX512NOTDQ-NEXT: movb %al, (%rsi)690; AVX512NOTDQ-NEXT: retq691 %d0 = load <2 x i1>, ptr %a0692 %d1 = shufflevector <2 x i1> %d0,<2 x i1> undef,<1 x i32><i32 1>693 store <1 x i1> %d1, ptr %a1694 ret void695}696define void @load_v3i1_broadcast_1_v1i1_store(ptr %a0,ptr %a1) {697; AVX512-LABEL: load_v3i1_broadcast_1_v1i1_store:698; AVX512: # %bb.0:699; AVX512-NEXT: movzbl (%rdi), %eax700; AVX512-NEXT: xorl %ecx, %ecx701; AVX512-NEXT: btl $1, %eax702; AVX512-NEXT: sbbl %ecx, %ecx703; AVX512-NEXT: kmovd %ecx, %k0704; AVX512-NEXT: kshiftrb $1, %k0, %k0705; AVX512-NEXT: kshiftlb $7, %k0, %k0706; AVX512-NEXT: kshiftrb $7, %k0, %k0707; AVX512-NEXT: kmovb %k0, (%rsi)708; AVX512-NEXT: retq709;710; AVX512NOTDQ-LABEL: load_v3i1_broadcast_1_v1i1_store:711; AVX512NOTDQ: # %bb.0:712; AVX512NOTDQ-NEXT: movzbl (%rdi), %eax713; AVX512NOTDQ-NEXT: xorl %ecx, %ecx714; AVX512NOTDQ-NEXT: btl $1, %eax715; AVX512NOTDQ-NEXT: sbbl %ecx, %ecx716; AVX512NOTDQ-NEXT: kmovd %ecx, %k0717; AVX512NOTDQ-NEXT: kshiftrw $1, %k0, %k0718; AVX512NOTDQ-NEXT: kshiftlw $15, %k0, %k0719; AVX512NOTDQ-NEXT: kshiftrw $15, %k0, %k0720; AVX512NOTDQ-NEXT: kmovd %k0, %eax721; AVX512NOTDQ-NEXT: movb %al, (%rsi)722; AVX512NOTDQ-NEXT: retq723 %d0 = load <3 x i1>, ptr %a0724 %d1 = shufflevector <3 x i1> %d0,<3 x i1> undef,<1 x i32><i32 1>725 store <1 x i1> %d1, ptr %a1726 ret void727}728define void @load_v3i1_broadcast_2_v1i1_store(ptr %a0,ptr %a1) {729; AVX512-LABEL: load_v3i1_broadcast_2_v1i1_store:730; AVX512: # %bb.0:731; AVX512-NEXT: movzbl (%rdi), %eax732; AVX512-NEXT: xorl %ecx, %ecx733; AVX512-NEXT: btl $2, %eax734; AVX512-NEXT: sbbl %ecx, %ecx735; AVX512-NEXT: kmovd %ecx, %k0736; AVX512-NEXT: kshiftrb $2, %k0, %k0737; AVX512-NEXT: kshiftlb $7, %k0, %k0738; AVX512-NEXT: kshiftrb $7, %k0, %k0739; AVX512-NEXT: kmovb %k0, (%rsi)740; AVX512-NEXT: retq741;742; AVX512NOTDQ-LABEL: load_v3i1_broadcast_2_v1i1_store:743; AVX512NOTDQ: # %bb.0:744; AVX512NOTDQ-NEXT: movzbl (%rdi), %eax745; AVX512NOTDQ-NEXT: xorl %ecx, %ecx746; AVX512NOTDQ-NEXT: btl $2, %eax747; AVX512NOTDQ-NEXT: sbbl %ecx, %ecx748; AVX512NOTDQ-NEXT: kmovd %ecx, %k0749; AVX512NOTDQ-NEXT: kshiftrw $2, %k0, %k0750; AVX512NOTDQ-NEXT: kshiftlw $15, %k0, %k0751; AVX512NOTDQ-NEXT: kshiftrw $15, %k0, %k0752; AVX512NOTDQ-NEXT: kmovd %k0, %eax753; AVX512NOTDQ-NEXT: movb %al, (%rsi)754; AVX512NOTDQ-NEXT: retq755 %d0 = load <3 x i1>, ptr %a0756 %d1 = shufflevector <3 x i1> %d0,<3 x i1> undef,<1 x i32><i32 2>757 store <1 x i1> %d1, ptr %a1758 ret void759}760define void @load_v4i1_broadcast_2_v1i1_store(ptr %a0,ptr %a1) {761; AVX512-LABEL: load_v4i1_broadcast_2_v1i1_store:762; AVX512: # %bb.0:763; AVX512-NEXT: kmovb (%rdi), %k0764; AVX512-NEXT: kshiftrb $2, %k0, %k0765; AVX512-NEXT: kshiftlb $7, %k0, %k0766; AVX512-NEXT: kshiftrb $7, %k0, %k0767; AVX512-NEXT: kmovb %k0, (%rsi)768; AVX512-NEXT: retq769;770; AVX512NOTDQ-LABEL: load_v4i1_broadcast_2_v1i1_store:771; AVX512NOTDQ: # %bb.0:772; AVX512NOTDQ-NEXT: movzbl (%rdi), %eax773; AVX512NOTDQ-NEXT: kmovd %eax, %k0774; AVX512NOTDQ-NEXT: kshiftrw $2, %k0, %k0775; AVX512NOTDQ-NEXT: kshiftlw $15, %k0, %k0776; AVX512NOTDQ-NEXT: kshiftrw $15, %k0, %k0777; AVX512NOTDQ-NEXT: kmovd %k0, %eax778; AVX512NOTDQ-NEXT: movb %al, (%rsi)779; AVX512NOTDQ-NEXT: retq780 %d0 = load <4 x i1>, ptr %a0781 %d1 = shufflevector <4 x i1> %d0,<4 x i1> undef,<1 x i32><i32 2>782 store <1 x i1> %d1, ptr %a1783 ret void784}785define void @load_v4i1_broadcast_3_v1i1_store(ptr %a0,ptr %a1) {786; AVX512-LABEL: load_v4i1_broadcast_3_v1i1_store:787; AVX512: # %bb.0:788; AVX512-NEXT: kmovb (%rdi), %k0789; AVX512-NEXT: kshiftrb $3, %k0, %k0790; AVX512-NEXT: kshiftlb $7, %k0, %k0791; AVX512-NEXT: kshiftrb $7, %k0, %k0792; AVX512-NEXT: kmovb %k0, (%rsi)793; AVX512-NEXT: retq794;795; AVX512NOTDQ-LABEL: load_v4i1_broadcast_3_v1i1_store:796; AVX512NOTDQ: # %bb.0:797; AVX512NOTDQ-NEXT: movzbl (%rdi), %eax798; AVX512NOTDQ-NEXT: kmovd %eax, %k0799; AVX512NOTDQ-NEXT: kshiftrw $3, %k0, %k0800; AVX512NOTDQ-NEXT: kshiftlw $15, %k0, %k0801; AVX512NOTDQ-NEXT: kshiftrw $15, %k0, %k0802; AVX512NOTDQ-NEXT: kmovd %k0, %eax803; AVX512NOTDQ-NEXT: movb %al, (%rsi)804; AVX512NOTDQ-NEXT: retq805 %d0 = load <4 x i1>, ptr %a0806 %d1 = shufflevector <4 x i1> %d0,<4 x i1> undef,<1 x i32><i32 3>807 store <1 x i1> %d1, ptr %a1808 ret void809}810define void @load_v8i1_broadcast_4_v1i1_store(ptr %a0,ptr %a1) {811; AVX512-LABEL: load_v8i1_broadcast_4_v1i1_store:812; AVX512: # %bb.0:813; AVX512-NEXT: kmovb (%rdi), %k0814; AVX512-NEXT: kshiftrb $4, %k0, %k0815; AVX512-NEXT: kshiftlb $7, %k0, %k0816; AVX512-NEXT: kshiftrb $7, %k0, %k0817; AVX512-NEXT: kmovb %k0, (%rsi)818; AVX512-NEXT: retq819;820; AVX512NOTDQ-LABEL: load_v8i1_broadcast_4_v1i1_store:821; AVX512NOTDQ: # %bb.0:822; AVX512NOTDQ-NEXT: movzbl (%rdi), %eax823; AVX512NOTDQ-NEXT: kmovd %eax, %k0824; AVX512NOTDQ-NEXT: kshiftrw $4, %k0, %k0825; AVX512NOTDQ-NEXT: kshiftlw $15, %k0, %k0826; AVX512NOTDQ-NEXT: kshiftrw $15, %k0, %k0827; AVX512NOTDQ-NEXT: kmovd %k0, %eax828; AVX512NOTDQ-NEXT: movb %al, (%rsi)829; AVX512NOTDQ-NEXT: retq830 %d0 = load <8 x i1>, ptr %a0831 %d1 = shufflevector <8 x i1> %d0,<8 x i1> undef,<1 x i32><i32 4>832 store <1 x i1> %d1, ptr %a1833 ret void834}835define void @load_v8i1_broadcast_4_v2i1_store(ptr %a0,ptr %a1) {836; AVX512-LABEL: load_v8i1_broadcast_4_v2i1_store:837; AVX512: # %bb.0:838; AVX512-NEXT: kmovb (%rdi), %k0839; AVX512-NEXT: kshiftrb $4, %k0, %k0840; AVX512-NEXT: vpmovm2q %k0, %xmm0841; AVX512-NEXT: vpbroadcastq %xmm0, %xmm0842; AVX512-NEXT: vpmovq2m %xmm0, %k0843; AVX512-NEXT: kmovb %k0, (%rsi)844; AVX512-NEXT: retq845;846; AVX512NOTDQ-LABEL: load_v8i1_broadcast_4_v2i1_store:847; AVX512NOTDQ: # %bb.0:848; AVX512NOTDQ-NEXT: movzbl (%rdi), %eax849; AVX512NOTDQ-NEXT: kmovd %eax, %k0850; AVX512NOTDQ-NEXT: kshiftrw $4, %k0, %k1851; AVX512NOTDQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0852; AVX512NOTDQ-NEXT: vmovdqa64 %xmm0, %xmm0 {%k1} {z}853; AVX512NOTDQ-NEXT: vpbroadcastq %xmm0, %xmm0854; AVX512NOTDQ-NEXT: vptestmq %xmm0, %xmm0, %k0855; AVX512NOTDQ-NEXT: kmovd %k0, %eax856; AVX512NOTDQ-NEXT: movb %al, (%rsi)857; AVX512NOTDQ-NEXT: retq858 %d0 = load <8 x i1>, ptr %a0859 %d1 = shufflevector <8 x i1> %d0,<8 x i1> undef,<2 x i32><i32 4,i32 4>860 store <2 x i1> %d1, ptr %a1861 ret void862}863define void @load_v8i1_broadcast_7_v1i1_store(ptr %a0,ptr %a1) {864; AVX512-LABEL: load_v8i1_broadcast_7_v1i1_store:865; AVX512: # %bb.0:866; AVX512-NEXT: kmovb (%rdi), %k0867; AVX512-NEXT: kshiftrb $7, %k0, %k0868; AVX512-NEXT: kshiftlb $7, %k0, %k0869; AVX512-NEXT: kshiftrb $7, %k0, %k0870; AVX512-NEXT: kmovb %k0, (%rsi)871; AVX512-NEXT: retq872;873; AVX512NOTDQ-LABEL: load_v8i1_broadcast_7_v1i1_store:874; AVX512NOTDQ: # %bb.0:875; AVX512NOTDQ-NEXT: movzbl (%rdi), %eax876; AVX512NOTDQ-NEXT: kmovd %eax, %k0877; AVX512NOTDQ-NEXT: kshiftrw $7, %k0, %k0878; AVX512NOTDQ-NEXT: kshiftlw $15, %k0, %k0879; AVX512NOTDQ-NEXT: kshiftrw $15, %k0, %k0880; AVX512NOTDQ-NEXT: kmovd %k0, %eax881; AVX512NOTDQ-NEXT: movb %al, (%rsi)882; AVX512NOTDQ-NEXT: retq883 %d0 = load <8 x i1>, ptr %a0884 %d1 = shufflevector <8 x i1> %d0,<8 x i1> undef,<1 x i32><i32 7>885 store <1 x i1> %d1, ptr %a1886 ret void887}888define void @load_v8i1_broadcast_7_v2i1_store(ptr %a0,ptr %a1) {889; AVX512-LABEL: load_v8i1_broadcast_7_v2i1_store:890; AVX512: # %bb.0:891; AVX512-NEXT: kmovb (%rdi), %k0892; AVX512-NEXT: kshiftrb $6, %k0, %k0893; AVX512-NEXT: vpmovm2q %k0, %xmm0894; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]895; AVX512-NEXT: vpmovq2m %xmm0, %k0896; AVX512-NEXT: kmovb %k0, (%rsi)897; AVX512-NEXT: retq898;899; AVX512NOTDQ-LABEL: load_v8i1_broadcast_7_v2i1_store:900; AVX512NOTDQ: # %bb.0:901; AVX512NOTDQ-NEXT: movzbl (%rdi), %eax902; AVX512NOTDQ-NEXT: kmovd %eax, %k0903; AVX512NOTDQ-NEXT: kshiftrw $6, %k0, %k1904; AVX512NOTDQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0905; AVX512NOTDQ-NEXT: vmovdqa64 %xmm0, %xmm0 {%k1} {z}906; AVX512NOTDQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]907; AVX512NOTDQ-NEXT: vptestmq %xmm0, %xmm0, %k0908; AVX512NOTDQ-NEXT: kmovd %k0, %eax909; AVX512NOTDQ-NEXT: movb %al, (%rsi)910; AVX512NOTDQ-NEXT: retq911 %d0 = load <8 x i1>, ptr %a0912 %d1 = shufflevector <8 x i1> %d0,<8 x i1> undef,<2 x i32><i32 7,i32 7>913 store <2 x i1> %d1, ptr %a1914 ret void915}916define void @load_v16i1_broadcast_8_v1i1_store(ptr %a0,ptr %a1) {917; AVX512-LABEL: load_v16i1_broadcast_8_v1i1_store:918; AVX512: # %bb.0:919; AVX512-NEXT: kmovw (%rdi), %k0920; AVX512-NEXT: kshiftrw $8, %k0, %k0921; AVX512-NEXT: kshiftlb $7, %k0, %k0922; AVX512-NEXT: kshiftrb $7, %k0, %k0923; AVX512-NEXT: kmovb %k0, (%rsi)924; AVX512-NEXT: retq925;926; AVX512NOTDQ-LABEL: load_v16i1_broadcast_8_v1i1_store:927; AVX512NOTDQ: # %bb.0:928; AVX512NOTDQ-NEXT: kmovw (%rdi), %k0929; AVX512NOTDQ-NEXT: kshiftrw $8, %k0, %k0930; AVX512NOTDQ-NEXT: kshiftlw $15, %k0, %k0931; AVX512NOTDQ-NEXT: kshiftrw $15, %k0, %k0932; AVX512NOTDQ-NEXT: kmovd %k0, %eax933; AVX512NOTDQ-NEXT: movb %al, (%rsi)934; AVX512NOTDQ-NEXT: retq935 %d0 = load <16 x i1>, ptr %a0936 %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<1 x i32><i32 8>937 store <1 x i1> %d1, ptr %a1938 ret void939}940define void @load_v16i1_broadcast_8_v2i1_store(ptr %a0,ptr %a1) {941; AVX512-LABEL: load_v16i1_broadcast_8_v2i1_store:942; AVX512: # %bb.0:943; AVX512-NEXT: kmovw (%rdi), %k0944; AVX512-NEXT: kshiftrw $8, %k0, %k0945; AVX512-NEXT: vpmovm2q %k0, %xmm0946; AVX512-NEXT: vpbroadcastq %xmm0, %xmm0947; AVX512-NEXT: vpmovq2m %xmm0, %k0948; AVX512-NEXT: kmovb %k0, (%rsi)949; AVX512-NEXT: retq950;951; AVX512NOTDQ-LABEL: load_v16i1_broadcast_8_v2i1_store:952; AVX512NOTDQ: # %bb.0:953; AVX512NOTDQ-NEXT: kmovw (%rdi), %k0954; AVX512NOTDQ-NEXT: kshiftrw $8, %k0, %k1955; AVX512NOTDQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0956; AVX512NOTDQ-NEXT: vmovdqa64 %xmm0, %xmm0 {%k1} {z}957; AVX512NOTDQ-NEXT: vpbroadcastq %xmm0, %xmm0958; AVX512NOTDQ-NEXT: vptestmq %xmm0, %xmm0, %k0959; AVX512NOTDQ-NEXT: kmovd %k0, %eax960; AVX512NOTDQ-NEXT: movb %al, (%rsi)961; AVX512NOTDQ-NEXT: retq962 %d0 = load <16 x i1>, ptr %a0963 %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<2 x i32><i32 8,i32 8>964 store <2 x i1> %d1, ptr %a1965 ret void966}967define void @load_v16i1_broadcast_8_v4i1_store(ptr %a0,ptr %a1) {968; AVX512-LABEL: load_v16i1_broadcast_8_v4i1_store:969; AVX512: # %bb.0:970; AVX512-NEXT: kmovw (%rdi), %k0971; AVX512-NEXT: kshiftrw $8, %k0, %k0972; AVX512-NEXT: vpmovm2d %k0, %xmm0973; AVX512-NEXT: vpbroadcastd %xmm0, %xmm0974; AVX512-NEXT: vpmovd2m %xmm0, %k0975; AVX512-NEXT: kmovb %k0, (%rsi)976; AVX512-NEXT: retq977;978; AVX512NOTDQ-LABEL: load_v16i1_broadcast_8_v4i1_store:979; AVX512NOTDQ: # %bb.0:980; AVX512NOTDQ-NEXT: kmovw (%rdi), %k0981; AVX512NOTDQ-NEXT: kshiftrw $8, %k0, %k1982; AVX512NOTDQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0983; AVX512NOTDQ-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}984; AVX512NOTDQ-NEXT: vpbroadcastd %xmm0, %xmm0985; AVX512NOTDQ-NEXT: vptestmd %xmm0, %xmm0, %k0986; AVX512NOTDQ-NEXT: kmovd %k0, %eax987; AVX512NOTDQ-NEXT: movb %al, (%rsi)988; AVX512NOTDQ-NEXT: retq989 %d0 = load <16 x i1>, ptr %a0990 %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<4 x i32><i32 8,i32 8,i32 8,i32 8>991 store <4 x i1> %d1, ptr %a1992 ret void993}994define void @load_v16i1_broadcast_15_v1i1_store(ptr %a0,ptr %a1) {995; AVX512-LABEL: load_v16i1_broadcast_15_v1i1_store:996; AVX512: # %bb.0:997; AVX512-NEXT: kmovw (%rdi), %k0998; AVX512-NEXT: kshiftrw $15, %k0, %k0999; AVX512-NEXT: kshiftlb $7, %k0, %k01000; AVX512-NEXT: kshiftrb $7, %k0, %k01001; AVX512-NEXT: kmovb %k0, (%rsi)1002; AVX512-NEXT: retq1003;1004; AVX512NOTDQ-LABEL: load_v16i1_broadcast_15_v1i1_store:1005; AVX512NOTDQ: # %bb.0:1006; AVX512NOTDQ-NEXT: kmovw (%rdi), %k01007; AVX512NOTDQ-NEXT: kshiftrw $15, %k0, %k01008; AVX512NOTDQ-NEXT: kshiftlw $15, %k0, %k01009; AVX512NOTDQ-NEXT: kshiftrw $15, %k0, %k01010; AVX512NOTDQ-NEXT: kmovd %k0, %eax1011; AVX512NOTDQ-NEXT: movb %al, (%rsi)1012; AVX512NOTDQ-NEXT: retq1013 %d0 = load <16 x i1>, ptr %a01014 %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<1 x i32><i32 15>1015 store <1 x i1> %d1, ptr %a11016 ret void1017}1018define void @load_v16i1_broadcast_15_v2i1_store(ptr %a0,ptr %a1) {1019; AVX512-LABEL: load_v16i1_broadcast_15_v2i1_store:1020; AVX512: # %bb.0:1021; AVX512-NEXT: kmovw (%rdi), %k01022; AVX512-NEXT: kshiftrw $14, %k0, %k01023; AVX512-NEXT: vpmovm2q %k0, %xmm01024; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]1025; AVX512-NEXT: vpmovq2m %xmm0, %k01026; AVX512-NEXT: kmovb %k0, (%rsi)1027; AVX512-NEXT: retq1028;1029; AVX512NOTDQ-LABEL: load_v16i1_broadcast_15_v2i1_store:1030; AVX512NOTDQ: # %bb.0:1031; AVX512NOTDQ-NEXT: kmovw (%rdi), %k01032; AVX512NOTDQ-NEXT: kshiftrw $14, %k0, %k11033; AVX512NOTDQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm01034; AVX512NOTDQ-NEXT: vmovdqa64 %xmm0, %xmm0 {%k1} {z}1035; AVX512NOTDQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]1036; AVX512NOTDQ-NEXT: vptestmq %xmm0, %xmm0, %k01037; AVX512NOTDQ-NEXT: kmovd %k0, %eax1038; AVX512NOTDQ-NEXT: movb %al, (%rsi)1039; AVX512NOTDQ-NEXT: retq1040 %d0 = load <16 x i1>, ptr %a01041 %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<2 x i32><i32 15,i32 15>1042 store <2 x i1> %d1, ptr %a11043 ret void1044}1045define void @load_v16i1_broadcast_15_v4i1_store(ptr %a0,ptr %a1) {1046; AVX512-LABEL: load_v16i1_broadcast_15_v4i1_store:1047; AVX512: # %bb.0:1048; AVX512-NEXT: kmovw (%rdi), %k01049; AVX512-NEXT: kshiftrw $12, %k0, %k01050; AVX512-NEXT: vpmovm2d %k0, %xmm01051; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]1052; AVX512-NEXT: vpmovd2m %xmm0, %k01053; AVX512-NEXT: kmovb %k0, (%rsi)1054; AVX512-NEXT: retq1055;1056; AVX512NOTDQ-LABEL: load_v16i1_broadcast_15_v4i1_store:1057; AVX512NOTDQ: # %bb.0:1058; AVX512NOTDQ-NEXT: kmovw (%rdi), %k01059; AVX512NOTDQ-NEXT: kshiftrw $12, %k0, %k11060; AVX512NOTDQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm01061; AVX512NOTDQ-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}1062; AVX512NOTDQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]1063; AVX512NOTDQ-NEXT: vptestmd %xmm0, %xmm0, %k01064; AVX512NOTDQ-NEXT: kmovd %k0, %eax1065; AVX512NOTDQ-NEXT: movb %al, (%rsi)1066; AVX512NOTDQ-NEXT: retq1067 %d0 = load <16 x i1>, ptr %a01068 %d1 = shufflevector <16 x i1> %d0,<16 x i1> undef,<4 x i32><i32 15,i32 15,i32 15,i32 15>1069 store <4 x i1> %d1, ptr %a11070 ret void1071}1072define void @load_v32i1_broadcast_16_v1i1_store(ptr %a0,ptr %a1) {1073; AVX512-LABEL: load_v32i1_broadcast_16_v1i1_store:1074; AVX512: # %bb.0:1075; AVX512-NEXT: kmovd (%rdi), %k01076; AVX512-NEXT: kshiftrd $16, %k0, %k01077; AVX512-NEXT: kshiftlb $7, %k0, %k01078; AVX512-NEXT: kshiftrb $7, %k0, %k01079; AVX512-NEXT: kmovb %k0, (%rsi)1080; AVX512-NEXT: retq1081;1082; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v1i1_store:1083; AVX512NOTDQ: # %bb.0:1084; AVX512NOTDQ-NEXT: kmovd (%rdi), %k01085; AVX512NOTDQ-NEXT: kshiftrd $16, %k0, %k01086; AVX512NOTDQ-NEXT: kshiftlw $15, %k0, %k01087; AVX512NOTDQ-NEXT: kshiftrw $15, %k0, %k01088; AVX512NOTDQ-NEXT: kmovd %k0, %eax1089; AVX512NOTDQ-NEXT: movb %al, (%rsi)1090; AVX512NOTDQ-NEXT: retq1091 %d0 = load <32 x i1>, ptr %a01092 %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<1 x i32><i32 16>1093 store <1 x i1> %d1, ptr %a11094 ret void1095}1096define void @load_v32i1_broadcast_16_v2i1_store(ptr %a0,ptr %a1) {1097; AVX512-LABEL: load_v32i1_broadcast_16_v2i1_store:1098; AVX512: # %bb.0:1099; AVX512-NEXT: kmovd (%rdi), %k01100; AVX512-NEXT: kshiftrd $16, %k0, %k01101; AVX512-NEXT: vpmovm2q %k0, %xmm01102; AVX512-NEXT: vpbroadcastq %xmm0, %xmm01103; AVX512-NEXT: vpmovq2m %xmm0, %k01104; AVX512-NEXT: kmovb %k0, (%rsi)1105; AVX512-NEXT: retq1106;1107; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v2i1_store:1108; AVX512NOTDQ: # %bb.0:1109; AVX512NOTDQ-NEXT: kmovd (%rdi), %k01110; AVX512NOTDQ-NEXT: kshiftrd $16, %k0, %k11111; AVX512NOTDQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm01112; AVX512NOTDQ-NEXT: vmovdqa64 %xmm0, %xmm0 {%k1} {z}1113; AVX512NOTDQ-NEXT: vpbroadcastq %xmm0, %xmm01114; AVX512NOTDQ-NEXT: vptestmq %xmm0, %xmm0, %k01115; AVX512NOTDQ-NEXT: kmovd %k0, %eax1116; AVX512NOTDQ-NEXT: movb %al, (%rsi)1117; AVX512NOTDQ-NEXT: retq1118 %d0 = load <32 x i1>, ptr %a01119 %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<2 x i32><i32 16,i32 16>1120 store <2 x i1> %d1, ptr %a11121 ret void1122}1123define void @load_v32i1_broadcast_16_v4i1_store(ptr %a0,ptr %a1) {1124; AVX512-LABEL: load_v32i1_broadcast_16_v4i1_store:1125; AVX512: # %bb.0:1126; AVX512-NEXT: kmovd (%rdi), %k01127; AVX512-NEXT: kshiftrd $16, %k0, %k01128; AVX512-NEXT: vpmovm2d %k0, %xmm01129; AVX512-NEXT: vpbroadcastd %xmm0, %xmm01130; AVX512-NEXT: vpmovd2m %xmm0, %k01131; AVX512-NEXT: kmovb %k0, (%rsi)1132; AVX512-NEXT: retq1133;1134; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v4i1_store:1135; AVX512NOTDQ: # %bb.0:1136; AVX512NOTDQ-NEXT: kmovd (%rdi), %k01137; AVX512NOTDQ-NEXT: kshiftrd $16, %k0, %k11138; AVX512NOTDQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm01139; AVX512NOTDQ-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}1140; AVX512NOTDQ-NEXT: vpbroadcastd %xmm0, %xmm01141; AVX512NOTDQ-NEXT: vptestmd %xmm0, %xmm0, %k01142; AVX512NOTDQ-NEXT: kmovd %k0, %eax1143; AVX512NOTDQ-NEXT: movb %al, (%rsi)1144; AVX512NOTDQ-NEXT: retq1145 %d0 = load <32 x i1>, ptr %a01146 %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<4 x i32><i32 16,i32 16,i32 16,i32 16>1147 store <4 x i1> %d1, ptr %a11148 ret void1149}1150define void @load_v32i1_broadcast_16_v8i1_store(ptr %a0,ptr %a1) {1151; AVX512-LABEL: load_v32i1_broadcast_16_v8i1_store:1152; AVX512: # %bb.0:1153; AVX512-NEXT: kmovb 2(%rdi), %k01154; AVX512-NEXT: vpmovm2d %k0, %ymm01155; AVX512-NEXT: vpbroadcastd %xmm0, %ymm01156; AVX512-NEXT: vpmovd2m %ymm0, %k01157; AVX512-NEXT: kmovb %k0, (%rsi)1158; AVX512-NEXT: vzeroupper1159; AVX512-NEXT: retq1160;1161; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v8i1_store:1162; AVX512NOTDQ: # %bb.0:1163; AVX512NOTDQ-NEXT: kmovw 2(%rdi), %k11164; AVX512NOTDQ-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm01165; AVX512NOTDQ-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z}1166; AVX512NOTDQ-NEXT: vpbroadcastd %xmm0, %ymm01167; AVX512NOTDQ-NEXT: vptestmd %ymm0, %ymm0, %k01168; AVX512NOTDQ-NEXT: kmovd %k0, %eax1169; AVX512NOTDQ-NEXT: movb %al, (%rsi)1170; AVX512NOTDQ-NEXT: vzeroupper1171; AVX512NOTDQ-NEXT: retq1172 %d0 = load <32 x i1>, ptr %a01173 %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<8 x i32><i32 16,i32 16,i32 16,i32 16,i32 16,i32 16,i32 16,i32 16>1174 store <8 x i1> %d1, ptr %a11175 ret void1176}1177define void @load_v32i1_broadcast_31_v1i1_store(ptr %a0,ptr %a1) {1178; AVX512-LABEL: load_v32i1_broadcast_31_v1i1_store:1179; AVX512: # %bb.0:1180; AVX512-NEXT: kmovd (%rdi), %k01181; AVX512-NEXT: kshiftrd $31, %k0, %k01182; AVX512-NEXT: kshiftlb $7, %k0, %k01183; AVX512-NEXT: kshiftrb $7, %k0, %k01184; AVX512-NEXT: kmovb %k0, (%rsi)1185; AVX512-NEXT: retq1186;1187; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v1i1_store:1188; AVX512NOTDQ: # %bb.0:1189; AVX512NOTDQ-NEXT: kmovd (%rdi), %k01190; AVX512NOTDQ-NEXT: kshiftrd $31, %k0, %k01191; AVX512NOTDQ-NEXT: kshiftlw $15, %k0, %k01192; AVX512NOTDQ-NEXT: kshiftrw $15, %k0, %k01193; AVX512NOTDQ-NEXT: kmovd %k0, %eax1194; AVX512NOTDQ-NEXT: movb %al, (%rsi)1195; AVX512NOTDQ-NEXT: retq1196 %d0 = load <32 x i1>, ptr %a01197 %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<1 x i32><i32 31>1198 store <1 x i1> %d1, ptr %a11199 ret void1200}1201define void @load_v32i1_broadcast_31_v2i1_store(ptr %a0,ptr %a1) {1202; AVX512-LABEL: load_v32i1_broadcast_31_v2i1_store:1203; AVX512: # %bb.0:1204; AVX512-NEXT: kmovd (%rdi), %k01205; AVX512-NEXT: kshiftrd $30, %k0, %k01206; AVX512-NEXT: vpmovm2q %k0, %xmm01207; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]1208; AVX512-NEXT: vpmovq2m %xmm0, %k01209; AVX512-NEXT: kmovb %k0, (%rsi)1210; AVX512-NEXT: retq1211;1212; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v2i1_store:1213; AVX512NOTDQ: # %bb.0:1214; AVX512NOTDQ-NEXT: kmovd (%rdi), %k01215; AVX512NOTDQ-NEXT: kshiftrd $30, %k0, %k11216; AVX512NOTDQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm01217; AVX512NOTDQ-NEXT: vmovdqa64 %xmm0, %xmm0 {%k1} {z}1218; AVX512NOTDQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]1219; AVX512NOTDQ-NEXT: vptestmq %xmm0, %xmm0, %k01220; AVX512NOTDQ-NEXT: kmovd %k0, %eax1221; AVX512NOTDQ-NEXT: movb %al, (%rsi)1222; AVX512NOTDQ-NEXT: retq1223 %d0 = load <32 x i1>, ptr %a01224 %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<2 x i32><i32 31,i32 31>1225 store <2 x i1> %d1, ptr %a11226 ret void1227}1228define void @load_v32i1_broadcast_31_v4i1_store(ptr %a0,ptr %a1) {1229; AVX512-LABEL: load_v32i1_broadcast_31_v4i1_store:1230; AVX512: # %bb.0:1231; AVX512-NEXT: kmovd (%rdi), %k01232; AVX512-NEXT: kshiftrd $28, %k0, %k01233; AVX512-NEXT: vpmovm2d %k0, %xmm01234; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]1235; AVX512-NEXT: vpmovd2m %xmm0, %k01236; AVX512-NEXT: kmovb %k0, (%rsi)1237; AVX512-NEXT: retq1238;1239; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v4i1_store:1240; AVX512NOTDQ: # %bb.0:1241; AVX512NOTDQ-NEXT: kmovd (%rdi), %k01242; AVX512NOTDQ-NEXT: kshiftrd $28, %k0, %k11243; AVX512NOTDQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm01244; AVX512NOTDQ-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}1245; AVX512NOTDQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]1246; AVX512NOTDQ-NEXT: vptestmd %xmm0, %xmm0, %k01247; AVX512NOTDQ-NEXT: kmovd %k0, %eax1248; AVX512NOTDQ-NEXT: movb %al, (%rsi)1249; AVX512NOTDQ-NEXT: retq1250 %d0 = load <32 x i1>, ptr %a01251 %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<4 x i32><i32 31,i32 31,i32 31,i32 31>1252 store <4 x i1> %d1, ptr %a11253 ret void1254}1255define void @load_v32i1_broadcast_31_v8i1_store(ptr %a0,ptr %a1) {1256; AVX512-FAST-LABEL: load_v32i1_broadcast_31_v8i1_store:1257; AVX512-FAST: # %bb.0:1258; AVX512-FAST-NEXT: kmovb 3(%rdi), %k01259; AVX512-FAST-NEXT: vpmovm2d %k0, %ymm01260; AVX512-FAST-NEXT: vpbroadcastd {{.*#+}} ymm1 = [7,7,7,7,7,7,7,7]1261; AVX512-FAST-NEXT: vpermd %ymm0, %ymm1, %ymm01262; AVX512-FAST-NEXT: vpmovd2m %ymm0, %k01263; AVX512-FAST-NEXT: kmovb %k0, (%rsi)1264; AVX512-FAST-NEXT: vzeroupper1265; AVX512-FAST-NEXT: retq1266;1267; AVX512-FAST-PERLANE-LABEL: load_v32i1_broadcast_31_v8i1_store:1268; AVX512-FAST-PERLANE: # %bb.0:1269; AVX512-FAST-PERLANE-NEXT: kmovb 3(%rdi), %k01270; AVX512-FAST-PERLANE-NEXT: vpmovm2d %k0, %ymm01271; AVX512-FAST-PERLANE-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[3,3,3,3,7,7,7,7]1272; AVX512-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]1273; AVX512-FAST-PERLANE-NEXT: vpmovd2m %ymm0, %k01274; AVX512-FAST-PERLANE-NEXT: kmovb %k0, (%rsi)1275; AVX512-FAST-PERLANE-NEXT: vzeroupper1276; AVX512-FAST-PERLANE-NEXT: retq1277;1278; AVX512NOTDQ-FAST-LABEL: load_v32i1_broadcast_31_v8i1_store:1279; AVX512NOTDQ-FAST: # %bb.0:1280; AVX512NOTDQ-FAST-NEXT: movzbl 3(%rdi), %eax1281; AVX512NOTDQ-FAST-NEXT: kmovd %eax, %k11282; AVX512NOTDQ-FAST-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm01283; AVX512NOTDQ-FAST-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z}1284; AVX512NOTDQ-FAST-NEXT: vpbroadcastd {{.*#+}} ymm1 = [7,7,7,7,7,7,7,7]1285; AVX512NOTDQ-FAST-NEXT: vpermd %ymm0, %ymm1, %ymm01286; AVX512NOTDQ-FAST-NEXT: vptestmd %ymm0, %ymm0, %k01287; AVX512NOTDQ-FAST-NEXT: kmovd %k0, %eax1288; AVX512NOTDQ-FAST-NEXT: movb %al, (%rsi)1289; AVX512NOTDQ-FAST-NEXT: vzeroupper1290; AVX512NOTDQ-FAST-NEXT: retq1291;1292; AVX512NOTDQ-FAST-PERLANE-LABEL: load_v32i1_broadcast_31_v8i1_store:1293; AVX512NOTDQ-FAST-PERLANE: # %bb.0:1294; AVX512NOTDQ-FAST-PERLANE-NEXT: movzbl 3(%rdi), %eax1295; AVX512NOTDQ-FAST-PERLANE-NEXT: kmovd %eax, %k11296; AVX512NOTDQ-FAST-PERLANE-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm01297; AVX512NOTDQ-FAST-PERLANE-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z}1298; AVX512NOTDQ-FAST-PERLANE-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[3,3,3,3,7,7,7,7]1299; AVX512NOTDQ-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]1300; AVX512NOTDQ-FAST-PERLANE-NEXT: vptestmd %ymm0, %ymm0, %k01301; AVX512NOTDQ-FAST-PERLANE-NEXT: kmovd %k0, %eax1302; AVX512NOTDQ-FAST-PERLANE-NEXT: movb %al, (%rsi)1303; AVX512NOTDQ-FAST-PERLANE-NEXT: vzeroupper1304; AVX512NOTDQ-FAST-PERLANE-NEXT: retq1305 %d0 = load <32 x i1>, ptr %a01306 %d1 = shufflevector <32 x i1> %d0,<32 x i1> undef,<8 x i32><i32 31,i32 31,i32 31,i32 31,i32 31,i32 31,i32 31,i32 31>1307 store <8 x i1> %d1, ptr %a11308 ret void1309}1310define void @load_v64i1_broadcast_32_v1i1_store(ptr %a0,ptr %a1) {1311; AVX512-LABEL: load_v64i1_broadcast_32_v1i1_store:1312; AVX512: # %bb.0:1313; AVX512-NEXT: kmovq (%rdi), %k01314; AVX512-NEXT: kshiftrq $32, %k0, %k01315; AVX512-NEXT: kshiftlb $7, %k0, %k01316; AVX512-NEXT: kshiftrb $7, %k0, %k01317; AVX512-NEXT: kmovb %k0, (%rsi)1318; AVX512-NEXT: retq1319;1320; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v1i1_store:1321; AVX512NOTDQ: # %bb.0:1322; AVX512NOTDQ-NEXT: kmovq (%rdi), %k01323; AVX512NOTDQ-NEXT: kshiftrq $32, %k0, %k01324; AVX512NOTDQ-NEXT: kshiftlw $15, %k0, %k01325; AVX512NOTDQ-NEXT: kshiftrw $15, %k0, %k01326; AVX512NOTDQ-NEXT: kmovd %k0, %eax1327; AVX512NOTDQ-NEXT: movb %al, (%rsi)1328; AVX512NOTDQ-NEXT: retq1329 %d0 = load <64 x i1>, ptr %a01330 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<1 x i32><i32 32>1331 store <1 x i1> %d1, ptr %a11332 ret void1333}1334define void @load_v64i1_broadcast_32_v2i1_store(ptr %a0,ptr %a1) {1335; AVX512-LABEL: load_v64i1_broadcast_32_v2i1_store:1336; AVX512: # %bb.0:1337; AVX512-NEXT: kmovq (%rdi), %k01338; AVX512-NEXT: kshiftrq $32, %k0, %k01339; AVX512-NEXT: vpmovm2q %k0, %xmm01340; AVX512-NEXT: vpbroadcastq %xmm0, %xmm01341; AVX512-NEXT: vpmovq2m %xmm0, %k01342; AVX512-NEXT: kmovb %k0, (%rsi)1343; AVX512-NEXT: retq1344;1345; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v2i1_store:1346; AVX512NOTDQ: # %bb.0:1347; AVX512NOTDQ-NEXT: kmovq (%rdi), %k01348; AVX512NOTDQ-NEXT: kshiftrq $32, %k0, %k11349; AVX512NOTDQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm01350; AVX512NOTDQ-NEXT: vmovdqa64 %xmm0, %xmm0 {%k1} {z}1351; AVX512NOTDQ-NEXT: vpbroadcastq %xmm0, %xmm01352; AVX512NOTDQ-NEXT: vptestmq %xmm0, %xmm0, %k01353; AVX512NOTDQ-NEXT: kmovd %k0, %eax1354; AVX512NOTDQ-NEXT: movb %al, (%rsi)1355; AVX512NOTDQ-NEXT: retq1356 %d0 = load <64 x i1>, ptr %a01357 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<2 x i32><i32 32,i32 32>1358 store <2 x i1> %d1, ptr %a11359 ret void1360}1361define void @load_v64i1_broadcast_32_v4i1_store(ptr %a0,ptr %a1) {1362; AVX512-LABEL: load_v64i1_broadcast_32_v4i1_store:1363; AVX512: # %bb.0:1364; AVX512-NEXT: kmovq (%rdi), %k01365; AVX512-NEXT: kshiftrq $32, %k0, %k01366; AVX512-NEXT: vpmovm2d %k0, %xmm01367; AVX512-NEXT: vpbroadcastd %xmm0, %xmm01368; AVX512-NEXT: vpmovd2m %xmm0, %k01369; AVX512-NEXT: kmovb %k0, (%rsi)1370; AVX512-NEXT: retq1371;1372; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v4i1_store:1373; AVX512NOTDQ: # %bb.0:1374; AVX512NOTDQ-NEXT: kmovq (%rdi), %k01375; AVX512NOTDQ-NEXT: kshiftrq $32, %k0, %k11376; AVX512NOTDQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm01377; AVX512NOTDQ-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}1378; AVX512NOTDQ-NEXT: vpbroadcastd %xmm0, %xmm01379; AVX512NOTDQ-NEXT: vptestmd %xmm0, %xmm0, %k01380; AVX512NOTDQ-NEXT: kmovd %k0, %eax1381; AVX512NOTDQ-NEXT: movb %al, (%rsi)1382; AVX512NOTDQ-NEXT: retq1383 %d0 = load <64 x i1>, ptr %a01384 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<4 x i32><i32 32,i32 32,i32 32,i32 32>1385 store <4 x i1> %d1, ptr %a11386 ret void1387}1388define void @load_v64i1_broadcast_32_v8i1_store(ptr %a0,ptr %a1) {1389; AVX512-LABEL: load_v64i1_broadcast_32_v8i1_store:1390; AVX512: # %bb.0:1391; AVX512-NEXT: kmovb 4(%rdi), %k01392; AVX512-NEXT: vpmovm2d %k0, %ymm01393; AVX512-NEXT: vpbroadcastd %xmm0, %ymm01394; AVX512-NEXT: vpmovd2m %ymm0, %k01395; AVX512-NEXT: kmovb %k0, (%rsi)1396; AVX512-NEXT: vzeroupper1397; AVX512-NEXT: retq1398;1399; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v8i1_store:1400; AVX512NOTDQ: # %bb.0:1401; AVX512NOTDQ-NEXT: kmovw 4(%rdi), %k11402; AVX512NOTDQ-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm01403; AVX512NOTDQ-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z}1404; AVX512NOTDQ-NEXT: vpbroadcastd %xmm0, %ymm01405; AVX512NOTDQ-NEXT: vptestmd %ymm0, %ymm0, %k01406; AVX512NOTDQ-NEXT: kmovd %k0, %eax1407; AVX512NOTDQ-NEXT: movb %al, (%rsi)1408; AVX512NOTDQ-NEXT: vzeroupper1409; AVX512NOTDQ-NEXT: retq1410 %d0 = load <64 x i1>, ptr %a01411 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<8 x i32><i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32>1412 store <8 x i1> %d1, ptr %a11413 ret void1414}1415define void @load_v64i1_broadcast_32_v16i1_store(ptr %a0,ptr %a1) {1416; AVX512-LABEL: load_v64i1_broadcast_32_v16i1_store:1417; AVX512: # %bb.0:1418; AVX512-NEXT: kmovw 4(%rdi), %k01419; AVX512-NEXT: vpmovm2d %k0, %zmm01420; AVX512-NEXT: vpbroadcastd %xmm0, %zmm01421; AVX512-NEXT: vpmovd2m %zmm0, %k01422; AVX512-NEXT: kmovw %k0, (%rsi)1423; AVX512-NEXT: vzeroupper1424; AVX512-NEXT: retq1425;1426; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v16i1_store:1427; AVX512NOTDQ: # %bb.0:1428; AVX512NOTDQ-NEXT: kmovw 4(%rdi), %k11429; AVX512NOTDQ-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -11430; AVX512NOTDQ-NEXT: vpbroadcastd %xmm0, %zmm01431; AVX512NOTDQ-NEXT: vptestmd %zmm0, %zmm0, %k01432; AVX512NOTDQ-NEXT: kmovw %k0, (%rsi)1433; AVX512NOTDQ-NEXT: vzeroupper1434; AVX512NOTDQ-NEXT: retq1435 %d0 = load <64 x i1>, ptr %a01436 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<16 x i32><i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32,i32 32>1437 store <16 x i1> %d1, ptr %a11438 ret void1439}1440define void @load_v64i1_broadcast_63_v1i1_store(ptr %a0,ptr %a1) {1441; AVX512-LABEL: load_v64i1_broadcast_63_v1i1_store:1442; AVX512: # %bb.0:1443; AVX512-NEXT: kmovq (%rdi), %k01444; AVX512-NEXT: kshiftrq $63, %k0, %k01445; AVX512-NEXT: kshiftlb $7, %k0, %k01446; AVX512-NEXT: kshiftrb $7, %k0, %k01447; AVX512-NEXT: kmovb %k0, (%rsi)1448; AVX512-NEXT: retq1449;1450; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v1i1_store:1451; AVX512NOTDQ: # %bb.0:1452; AVX512NOTDQ-NEXT: kmovq (%rdi), %k01453; AVX512NOTDQ-NEXT: kshiftrq $63, %k0, %k01454; AVX512NOTDQ-NEXT: kshiftlw $15, %k0, %k01455; AVX512NOTDQ-NEXT: kshiftrw $15, %k0, %k01456; AVX512NOTDQ-NEXT: kmovd %k0, %eax1457; AVX512NOTDQ-NEXT: movb %al, (%rsi)1458; AVX512NOTDQ-NEXT: retq1459 %d0 = load <64 x i1>, ptr %a01460 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<1 x i32><i32 63>1461 store <1 x i1> %d1, ptr %a11462 ret void1463}1464define void @load_v64i1_broadcast_63_v2i1_store(ptr %a0,ptr %a1) {1465; AVX512-LABEL: load_v64i1_broadcast_63_v2i1_store:1466; AVX512: # %bb.0:1467; AVX512-NEXT: kmovq (%rdi), %k01468; AVX512-NEXT: kshiftrq $62, %k0, %k01469; AVX512-NEXT: vpmovm2q %k0, %xmm01470; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]1471; AVX512-NEXT: vpmovq2m %xmm0, %k01472; AVX512-NEXT: kmovb %k0, (%rsi)1473; AVX512-NEXT: retq1474;1475; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v2i1_store:1476; AVX512NOTDQ: # %bb.0:1477; AVX512NOTDQ-NEXT: kmovq (%rdi), %k01478; AVX512NOTDQ-NEXT: kshiftrq $62, %k0, %k11479; AVX512NOTDQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm01480; AVX512NOTDQ-NEXT: vmovdqa64 %xmm0, %xmm0 {%k1} {z}1481; AVX512NOTDQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]1482; AVX512NOTDQ-NEXT: vptestmq %xmm0, %xmm0, %k01483; AVX512NOTDQ-NEXT: kmovd %k0, %eax1484; AVX512NOTDQ-NEXT: movb %al, (%rsi)1485; AVX512NOTDQ-NEXT: retq1486 %d0 = load <64 x i1>, ptr %a01487 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<2 x i32><i32 63,i32 63>1488 store <2 x i1> %d1, ptr %a11489 ret void1490}1491define void @load_v64i1_broadcast_63_v4i1_store(ptr %a0,ptr %a1) {1492; AVX512-LABEL: load_v64i1_broadcast_63_v4i1_store:1493; AVX512: # %bb.0:1494; AVX512-NEXT: kmovq (%rdi), %k01495; AVX512-NEXT: kshiftrq $60, %k0, %k01496; AVX512-NEXT: vpmovm2d %k0, %xmm01497; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]1498; AVX512-NEXT: vpmovd2m %xmm0, %k01499; AVX512-NEXT: kmovb %k0, (%rsi)1500; AVX512-NEXT: retq1501;1502; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v4i1_store:1503; AVX512NOTDQ: # %bb.0:1504; AVX512NOTDQ-NEXT: kmovq (%rdi), %k01505; AVX512NOTDQ-NEXT: kshiftrq $60, %k0, %k11506; AVX512NOTDQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm01507; AVX512NOTDQ-NEXT: vmovdqa32 %xmm0, %xmm0 {%k1} {z}1508; AVX512NOTDQ-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]1509; AVX512NOTDQ-NEXT: vptestmd %xmm0, %xmm0, %k01510; AVX512NOTDQ-NEXT: kmovd %k0, %eax1511; AVX512NOTDQ-NEXT: movb %al, (%rsi)1512; AVX512NOTDQ-NEXT: retq1513 %d0 = load <64 x i1>, ptr %a01514 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<4 x i32><i32 63,i32 63,i32 63,i32 63>1515 store <4 x i1> %d1, ptr %a11516 ret void1517}1518define void @load_v64i1_broadcast_63_v8i1_store(ptr %a0,ptr %a1) {1519; AVX512-FAST-LABEL: load_v64i1_broadcast_63_v8i1_store:1520; AVX512-FAST: # %bb.0:1521; AVX512-FAST-NEXT: kmovb 7(%rdi), %k01522; AVX512-FAST-NEXT: vpmovm2d %k0, %ymm01523; AVX512-FAST-NEXT: vpbroadcastd {{.*#+}} ymm1 = [7,7,7,7,7,7,7,7]1524; AVX512-FAST-NEXT: vpermd %ymm0, %ymm1, %ymm01525; AVX512-FAST-NEXT: vpmovd2m %ymm0, %k01526; AVX512-FAST-NEXT: kmovb %k0, (%rsi)1527; AVX512-FAST-NEXT: vzeroupper1528; AVX512-FAST-NEXT: retq1529;1530; AVX512-FAST-PERLANE-LABEL: load_v64i1_broadcast_63_v8i1_store:1531; AVX512-FAST-PERLANE: # %bb.0:1532; AVX512-FAST-PERLANE-NEXT: kmovb 7(%rdi), %k01533; AVX512-FAST-PERLANE-NEXT: vpmovm2d %k0, %ymm01534; AVX512-FAST-PERLANE-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[3,3,3,3,7,7,7,7]1535; AVX512-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]1536; AVX512-FAST-PERLANE-NEXT: vpmovd2m %ymm0, %k01537; AVX512-FAST-PERLANE-NEXT: kmovb %k0, (%rsi)1538; AVX512-FAST-PERLANE-NEXT: vzeroupper1539; AVX512-FAST-PERLANE-NEXT: retq1540;1541; AVX512NOTDQ-FAST-LABEL: load_v64i1_broadcast_63_v8i1_store:1542; AVX512NOTDQ-FAST: # %bb.0:1543; AVX512NOTDQ-FAST-NEXT: movzbl 7(%rdi), %eax1544; AVX512NOTDQ-FAST-NEXT: kmovd %eax, %k11545; AVX512NOTDQ-FAST-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm01546; AVX512NOTDQ-FAST-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z}1547; AVX512NOTDQ-FAST-NEXT: vpbroadcastd {{.*#+}} ymm1 = [7,7,7,7,7,7,7,7]1548; AVX512NOTDQ-FAST-NEXT: vpermd %ymm0, %ymm1, %ymm01549; AVX512NOTDQ-FAST-NEXT: vptestmd %ymm0, %ymm0, %k01550; AVX512NOTDQ-FAST-NEXT: kmovd %k0, %eax1551; AVX512NOTDQ-FAST-NEXT: movb %al, (%rsi)1552; AVX512NOTDQ-FAST-NEXT: vzeroupper1553; AVX512NOTDQ-FAST-NEXT: retq1554;1555; AVX512NOTDQ-FAST-PERLANE-LABEL: load_v64i1_broadcast_63_v8i1_store:1556; AVX512NOTDQ-FAST-PERLANE: # %bb.0:1557; AVX512NOTDQ-FAST-PERLANE-NEXT: movzbl 7(%rdi), %eax1558; AVX512NOTDQ-FAST-PERLANE-NEXT: kmovd %eax, %k11559; AVX512NOTDQ-FAST-PERLANE-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm01560; AVX512NOTDQ-FAST-PERLANE-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z}1561; AVX512NOTDQ-FAST-PERLANE-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[3,3,3,3,7,7,7,7]1562; AVX512NOTDQ-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,2,2]1563; AVX512NOTDQ-FAST-PERLANE-NEXT: vptestmd %ymm0, %ymm0, %k01564; AVX512NOTDQ-FAST-PERLANE-NEXT: kmovd %k0, %eax1565; AVX512NOTDQ-FAST-PERLANE-NEXT: movb %al, (%rsi)1566; AVX512NOTDQ-FAST-PERLANE-NEXT: vzeroupper1567; AVX512NOTDQ-FAST-PERLANE-NEXT: retq1568 %d0 = load <64 x i1>, ptr %a01569 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<8 x i32><i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63>1570 store <8 x i1> %d1, ptr %a11571 ret void1572}1573define void @load_v64i1_broadcast_63_v16i1_store(ptr %a0,ptr %a1) {1574; AVX512-FAST-LABEL: load_v64i1_broadcast_63_v16i1_store:1575; AVX512-FAST: # %bb.0:1576; AVX512-FAST-NEXT: kmovw 6(%rdi), %k01577; AVX512-FAST-NEXT: vpmovm2d %k0, %zmm01578; AVX512-FAST-NEXT: vpbroadcastd {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]1579; AVX512-FAST-NEXT: vpermd %zmm0, %zmm1, %zmm01580; AVX512-FAST-NEXT: vpmovd2m %zmm0, %k01581; AVX512-FAST-NEXT: kmovw %k0, (%rsi)1582; AVX512-FAST-NEXT: vzeroupper1583; AVX512-FAST-NEXT: retq1584;1585; AVX512-FAST-PERLANE-LABEL: load_v64i1_broadcast_63_v16i1_store:1586; AVX512-FAST-PERLANE: # %bb.0:1587; AVX512-FAST-PERLANE-NEXT: kmovw 6(%rdi), %k01588; AVX512-FAST-PERLANE-NEXT: vpmovm2d %k0, %zmm01589; AVX512-FAST-PERLANE-NEXT: vpshufd {{.*#+}} zmm0 = zmm0[3,3,3,3,7,7,7,7,11,11,11,11,15,15,15,15]1590; AVX512-FAST-PERLANE-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[6,7,6,7,6,7,6,7]1591; AVX512-FAST-PERLANE-NEXT: vpmovd2m %zmm0, %k01592; AVX512-FAST-PERLANE-NEXT: kmovw %k0, (%rsi)1593; AVX512-FAST-PERLANE-NEXT: vzeroupper1594; AVX512-FAST-PERLANE-NEXT: retq1595;1596; AVX512NOTDQ-FAST-LABEL: load_v64i1_broadcast_63_v16i1_store:1597; AVX512NOTDQ-FAST: # %bb.0:1598; AVX512NOTDQ-FAST-NEXT: kmovw 6(%rdi), %k11599; AVX512NOTDQ-FAST-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -11600; AVX512NOTDQ-FAST-NEXT: vpbroadcastd {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]1601; AVX512NOTDQ-FAST-NEXT: vpermd %zmm0, %zmm1, %zmm01602; AVX512NOTDQ-FAST-NEXT: vptestmd %zmm0, %zmm0, %k01603; AVX512NOTDQ-FAST-NEXT: kmovw %k0, (%rsi)1604; AVX512NOTDQ-FAST-NEXT: vzeroupper1605; AVX512NOTDQ-FAST-NEXT: retq1606;1607; AVX512NOTDQ-FAST-PERLANE-LABEL: load_v64i1_broadcast_63_v16i1_store:1608; AVX512NOTDQ-FAST-PERLANE: # %bb.0:1609; AVX512NOTDQ-FAST-PERLANE-NEXT: kmovw 6(%rdi), %k11610; AVX512NOTDQ-FAST-PERLANE-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -11611; AVX512NOTDQ-FAST-PERLANE-NEXT: vpshufd {{.*#+}} zmm0 = zmm0[3,3,3,3,7,7,7,7,11,11,11,11,15,15,15,15]1612; AVX512NOTDQ-FAST-PERLANE-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[6,7,6,7,6,7,6,7]1613; AVX512NOTDQ-FAST-PERLANE-NEXT: vptestmd %zmm0, %zmm0, %k01614; AVX512NOTDQ-FAST-PERLANE-NEXT: kmovw %k0, (%rsi)1615; AVX512NOTDQ-FAST-PERLANE-NEXT: vzeroupper1616; AVX512NOTDQ-FAST-PERLANE-NEXT: retq1617 %d0 = load <64 x i1>, ptr %a01618 %d1 = shufflevector <64 x i1> %d0,<64 x i1> undef,<16 x i32><i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63,i32 63>1619 store <16 x i1> %d1, ptr %a11620 ret void1621}1622 1623