1541 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s -check-prefix=AVX3; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s -check-prefix=AVX24; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s -check-prefix=AVX5125 6define void @test_masked_store_success_v4i8(<4 x i8> %x, ptr %ptr, <4 x i1> %mask) {7; AVX-LABEL: test_masked_store_success_v4i8:8; AVX: # %bb.0:9; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]10; AVX-NEXT: vpsllw $7, %xmm1, %xmm111; AVX-NEXT: vmovdqa (%rdi), %xmm212; AVX-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm013; AVX-NEXT: vmovd %xmm0, (%rdi)14; AVX-NEXT: retq15;16; AVX2-LABEL: test_masked_store_success_v4i8:17; AVX2: # %bb.0:18; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]19; AVX2-NEXT: vpsllw $7, %xmm1, %xmm120; AVX2-NEXT: vmovdqa (%rdi), %xmm221; AVX2-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm022; AVX2-NEXT: vmovd %xmm0, (%rdi)23; AVX2-NEXT: retq24;25; AVX512-LABEL: test_masked_store_success_v4i8:26; AVX512: # %bb.0:27; AVX512-NEXT: vpslld $31, %xmm1, %xmm128; AVX512-NEXT: vpmovd2m %xmm1, %k129; AVX512-NEXT: vmovdqa (%rdi), %xmm130; AVX512-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1}31; AVX512-NEXT: vmovd %xmm1, (%rdi)32; AVX512-NEXT: retq33 %load = load <4 x i8>, ptr %ptr, align 3234 %sel = select <4 x i1> %mask, <4 x i8> %x, <4 x i8> %load35 store <4 x i8> %sel, ptr %ptr, align 3236 ret void37}38 39define void @test_masked_store_success_v4i16(<4 x i16> %x, ptr %ptr, <4 x i1> %mask) {40; AVX-LABEL: test_masked_store_success_v4i16:41; AVX: # %bb.0:42; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,u,4,u,8,u,12,u,8,u,12,u,12,u,14,u]43; AVX-NEXT: vpsllw $15, %xmm1, %xmm144; AVX-NEXT: vpsraw $15, %xmm1, %xmm145; AVX-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero46; AVX-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm047; AVX-NEXT: vmovq %xmm0, (%rdi)48; AVX-NEXT: retq49;50; AVX2-LABEL: test_masked_store_success_v4i16:51; AVX2: # %bb.0:52; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,u,4,u,8,u,12,u,8,u,12,u,12,u,14,u]53; AVX2-NEXT: vpsllw $15, %xmm1, %xmm154; AVX2-NEXT: vpsraw $15, %xmm1, %xmm155; AVX2-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero56; AVX2-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm057; AVX2-NEXT: vmovq %xmm0, (%rdi)58; AVX2-NEXT: retq59;60; AVX512-LABEL: test_masked_store_success_v4i16:61; AVX512: # %bb.0:62; AVX512-NEXT: vpslld $31, %xmm1, %xmm163; AVX512-NEXT: vpmovd2m %xmm1, %k164; AVX512-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero65; AVX512-NEXT: vmovdqu16 %xmm0, %xmm1 {%k1}66; AVX512-NEXT: vmovq %xmm1, (%rdi)67; AVX512-NEXT: retq68 %load = load <4 x i16>, ptr %ptr, align 3269 %sel = select <4 x i1> %mask, <4 x i16> %x, <4 x i16> %load70 store <4 x i16> %sel, ptr %ptr, align 3271 ret void72}73 74define void @test_masked_store_success_v4i32(<4 x i32> %x, ptr %ptr, <4 x i1> %mask) {75; AVX-LABEL: test_masked_store_success_v4i32:76; AVX: # %bb.0:77; AVX-NEXT: vpslld $31, %xmm1, %xmm178; AVX-NEXT: vmaskmovps %xmm0, %xmm1, (%rdi)79; AVX-NEXT: retq80;81; AVX2-LABEL: test_masked_store_success_v4i32:82; AVX2: # %bb.0:83; AVX2-NEXT: vpslld $31, %xmm1, %xmm184; AVX2-NEXT: vpmaskmovd %xmm0, %xmm1, (%rdi)85; AVX2-NEXT: retq86;87; AVX512-LABEL: test_masked_store_success_v4i32:88; AVX512: # %bb.0:89; AVX512-NEXT: vpslld $31, %xmm1, %xmm190; AVX512-NEXT: vpmovd2m %xmm1, %k191; AVX512-NEXT: vmovdqa32 %xmm0, (%rdi) {%k1}92; AVX512-NEXT: retq93 %load = load <4 x i32>, ptr %ptr, align 3294 %sel = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %load95 store <4 x i32> %sel, ptr %ptr, align 3296 ret void97}98 99define void @test_masked_store_success_v4i64(<4 x i64> %x, ptr %ptr, <4 x i1> %mask) {100; AVX-LABEL: test_masked_store_success_v4i64:101; AVX: # %bb.0:102; AVX-NEXT: vpslld $31, %xmm1, %xmm1103; AVX-NEXT: vpmovsxdq %xmm1, %xmm2104; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]105; AVX-NEXT: vpmovsxdq %xmm1, %xmm1106; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1107; AVX-NEXT: vmaskmovpd %ymm0, %ymm1, (%rdi)108; AVX-NEXT: vzeroupper109; AVX-NEXT: retq110;111; AVX2-LABEL: test_masked_store_success_v4i64:112; AVX2: # %bb.0:113; AVX2-NEXT: vpslld $31, %xmm1, %xmm1114; AVX2-NEXT: vpmovsxdq %xmm1, %ymm1115; AVX2-NEXT: vpmaskmovq %ymm0, %ymm1, (%rdi)116; AVX2-NEXT: vzeroupper117; AVX2-NEXT: retq118;119; AVX512-LABEL: test_masked_store_success_v4i64:120; AVX512: # %bb.0:121; AVX512-NEXT: vpslld $31, %xmm1, %xmm1122; AVX512-NEXT: vpmovd2m %xmm1, %k1123; AVX512-NEXT: vmovdqa64 %ymm0, (%rdi) {%k1}124; AVX512-NEXT: vzeroupper125; AVX512-NEXT: retq126 %load = load <4 x i64>, ptr %ptr, align 32127 %sel = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> %load128 store <4 x i64> %sel, ptr %ptr, align 32129 ret void130}131 132define void @test_masked_store_success_v4f16(<4 x half> %x, ptr %ptr, <4 x i1> %mask) {133; AVX-LABEL: test_masked_store_success_v4f16:134; AVX: # %bb.0:135; AVX-NEXT: vpsrlq $48, %xmm0, %xmm2136; AVX-NEXT: vpextrw $0, %xmm2, %edx137; AVX-NEXT: vpsrld $16, %xmm0, %xmm2138; AVX-NEXT: vpextrw $0, %xmm2, %ecx139; AVX-NEXT: movzwl 2(%rdi), %eax140; AVX-NEXT: vpextrb $4, %xmm1, %esi141; AVX-NEXT: testb $1, %sil142; AVX-NEXT: cmovnel %ecx, %eax143; AVX-NEXT: vpextrb $8, %xmm1, %ecx144; AVX-NEXT: testb $1, %cl145; AVX-NEXT: jne .LBB4_1146; AVX-NEXT: # %bb.2:147; AVX-NEXT: movl 4(%rdi), %ecx148; AVX-NEXT: jmp .LBB4_3149; AVX-NEXT: .LBB4_1:150; AVX-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]151; AVX-NEXT: vpextrw $0, %xmm2, %ecx152; AVX-NEXT: .LBB4_3:153; AVX-NEXT: movzwl 6(%rdi), %esi154; AVX-NEXT: vpextrb $12, %xmm1, %r8d155; AVX-NEXT: testb $1, %r8b156; AVX-NEXT: cmovnel %edx, %esi157; AVX-NEXT: vmovd %xmm1, %edx158; AVX-NEXT: testb $1, %dl159; AVX-NEXT: jne .LBB4_4160; AVX-NEXT: # %bb.5:161; AVX-NEXT: movl (%rdi), %edx162; AVX-NEXT: jmp .LBB4_6163; AVX-NEXT: .LBB4_4:164; AVX-NEXT: vpextrw $0, %xmm0, %edx165; AVX-NEXT: .LBB4_6:166; AVX-NEXT: movw %dx, (%rdi)167; AVX-NEXT: movw %si, 6(%rdi)168; AVX-NEXT: movw %cx, 4(%rdi)169; AVX-NEXT: movw %ax, 2(%rdi)170; AVX-NEXT: retq171;172; AVX2-LABEL: test_masked_store_success_v4f16:173; AVX2: # %bb.0:174; AVX2-NEXT: vpsrlq $48, %xmm0, %xmm2175; AVX2-NEXT: vpextrw $0, %xmm2, %edx176; AVX2-NEXT: vpsrld $16, %xmm0, %xmm2177; AVX2-NEXT: vpextrw $0, %xmm2, %ecx178; AVX2-NEXT: movzwl 2(%rdi), %eax179; AVX2-NEXT: vpextrb $4, %xmm1, %esi180; AVX2-NEXT: testb $1, %sil181; AVX2-NEXT: cmovnel %ecx, %eax182; AVX2-NEXT: vpextrb $8, %xmm1, %ecx183; AVX2-NEXT: testb $1, %cl184; AVX2-NEXT: jne .LBB4_1185; AVX2-NEXT: # %bb.2:186; AVX2-NEXT: movl 4(%rdi), %ecx187; AVX2-NEXT: jmp .LBB4_3188; AVX2-NEXT: .LBB4_1:189; AVX2-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]190; AVX2-NEXT: vpextrw $0, %xmm2, %ecx191; AVX2-NEXT: .LBB4_3:192; AVX2-NEXT: movzwl 6(%rdi), %esi193; AVX2-NEXT: vpextrb $12, %xmm1, %r8d194; AVX2-NEXT: testb $1, %r8b195; AVX2-NEXT: cmovnel %edx, %esi196; AVX2-NEXT: vmovd %xmm1, %edx197; AVX2-NEXT: testb $1, %dl198; AVX2-NEXT: jne .LBB4_4199; AVX2-NEXT: # %bb.5:200; AVX2-NEXT: movl (%rdi), %edx201; AVX2-NEXT: jmp .LBB4_6202; AVX2-NEXT: .LBB4_4:203; AVX2-NEXT: vpextrw $0, %xmm0, %edx204; AVX2-NEXT: .LBB4_6:205; AVX2-NEXT: movw %dx, (%rdi)206; AVX2-NEXT: movw %si, 6(%rdi)207; AVX2-NEXT: movw %cx, 4(%rdi)208; AVX2-NEXT: movw %ax, 2(%rdi)209; AVX2-NEXT: retq210;211; AVX512-LABEL: test_masked_store_success_v4f16:212; AVX512: # %bb.0:213; AVX512-NEXT: vpslld $31, %xmm1, %xmm1214; AVX512-NEXT: vpmovd2m %xmm1, %k1215; AVX512-NEXT: vmovdqa (%rdi), %xmm1216; AVX512-NEXT: vmovdqu16 %xmm0, %xmm1 {%k1}217; AVX512-NEXT: vmovq %xmm1, (%rdi)218; AVX512-NEXT: retq219 %load = load <4 x half>, ptr %ptr, align 32220 %sel = select <4 x i1> %mask, <4 x half> %x, <4 x half> %load221 store <4 x half> %sel, ptr %ptr, align 32222 ret void223}224 225define void @test_masked_store_success_v4f32(<4 x float> %x, ptr %ptr, <4 x i1> %mask) {226; AVX-LABEL: test_masked_store_success_v4f32:227; AVX: # %bb.0:228; AVX-NEXT: vpslld $31, %xmm1, %xmm1229; AVX-NEXT: vmaskmovps %xmm0, %xmm1, (%rdi)230; AVX-NEXT: retq231;232; AVX2-LABEL: test_masked_store_success_v4f32:233; AVX2: # %bb.0:234; AVX2-NEXT: vpslld $31, %xmm1, %xmm1235; AVX2-NEXT: vmaskmovps %xmm0, %xmm1, (%rdi)236; AVX2-NEXT: retq237;238; AVX512-LABEL: test_masked_store_success_v4f32:239; AVX512: # %bb.0:240; AVX512-NEXT: vpslld $31, %xmm1, %xmm1241; AVX512-NEXT: vpmovd2m %xmm1, %k1242; AVX512-NEXT: vmovaps %xmm0, (%rdi) {%k1}243; AVX512-NEXT: retq244 %load = load <4 x float>, ptr %ptr, align 32245 %sel = select <4 x i1> %mask, <4 x float> %x, <4 x float> %load246 store <4 x float> %sel, ptr %ptr, align 32247 ret void248}249 250define void @test_masked_store_success_v4f64(<4 x double> %x, ptr %ptr, <4 x i1> %mask) {251; AVX-LABEL: test_masked_store_success_v4f64:252; AVX: # %bb.0:253; AVX-NEXT: vpslld $31, %xmm1, %xmm1254; AVX-NEXT: vpmovsxdq %xmm1, %xmm2255; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]256; AVX-NEXT: vpmovsxdq %xmm1, %xmm1257; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1258; AVX-NEXT: vmaskmovpd %ymm0, %ymm1, (%rdi)259; AVX-NEXT: vzeroupper260; AVX-NEXT: retq261;262; AVX2-LABEL: test_masked_store_success_v4f64:263; AVX2: # %bb.0:264; AVX2-NEXT: vpslld $31, %xmm1, %xmm1265; AVX2-NEXT: vpmovsxdq %xmm1, %ymm1266; AVX2-NEXT: vmaskmovpd %ymm0, %ymm1, (%rdi)267; AVX2-NEXT: vzeroupper268; AVX2-NEXT: retq269;270; AVX512-LABEL: test_masked_store_success_v4f64:271; AVX512: # %bb.0:272; AVX512-NEXT: vpslld $31, %xmm1, %xmm1273; AVX512-NEXT: vpmovd2m %xmm1, %k1274; AVX512-NEXT: vmovapd %ymm0, (%rdi) {%k1}275; AVX512-NEXT: vzeroupper276; AVX512-NEXT: retq277 %load = load <4 x double>, ptr %ptr, align 32278 %sel = select <4 x i1> %mask, <4 x double> %x, <4 x double> %load279 store <4 x double> %sel, ptr %ptr, align 32280 ret void281}282 283define void @test_masked_store_success_v8i8(<8 x i8> %x, ptr %ptr, <8 x i1> %mask) {284; AVX-LABEL: test_masked_store_success_v8i8:285; AVX: # %bb.0:286; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]287; AVX-NEXT: vpsllw $7, %xmm1, %xmm1288; AVX-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero289; AVX-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm0290; AVX-NEXT: vmovq %xmm0, (%rdi)291; AVX-NEXT: retq292;293; AVX2-LABEL: test_masked_store_success_v8i8:294; AVX2: # %bb.0:295; AVX2-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]296; AVX2-NEXT: vpsllw $7, %xmm1, %xmm1297; AVX2-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero298; AVX2-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm0299; AVX2-NEXT: vmovq %xmm0, (%rdi)300; AVX2-NEXT: retq301;302; AVX512-LABEL: test_masked_store_success_v8i8:303; AVX512: # %bb.0:304; AVX512-NEXT: vpsllw $15, %xmm1, %xmm1305; AVX512-NEXT: vpmovw2m %xmm1, %k1306; AVX512-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero307; AVX512-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1}308; AVX512-NEXT: vmovq %xmm1, (%rdi)309; AVX512-NEXT: retq310 %load = load <8 x i8>, ptr %ptr, align 32311 %sel = select <8 x i1> %mask, <8 x i8> %x, <8 x i8> %load312 store <8 x i8> %sel, ptr %ptr, align 32313 ret void314}315 316define void @test_masked_store_success_v8i16(<8 x i16> %x, ptr %ptr, <8 x i1> %mask) {317; AVX-LABEL: test_masked_store_success_v8i16:318; AVX: # %bb.0:319; AVX-NEXT: vpsllw $15, %xmm1, %xmm1320; AVX-NEXT: vpsraw $15, %xmm1, %xmm1321; AVX-NEXT: vmovdqa (%rdi), %xmm2322; AVX-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm0323; AVX-NEXT: vmovdqa %xmm0, (%rdi)324; AVX-NEXT: retq325;326; AVX2-LABEL: test_masked_store_success_v8i16:327; AVX2: # %bb.0:328; AVX2-NEXT: vpsllw $15, %xmm1, %xmm1329; AVX2-NEXT: vpsraw $15, %xmm1, %xmm1330; AVX2-NEXT: vmovdqa (%rdi), %xmm2331; AVX2-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm0332; AVX2-NEXT: vmovdqa %xmm0, (%rdi)333; AVX2-NEXT: retq334;335; AVX512-LABEL: test_masked_store_success_v8i16:336; AVX512: # %bb.0:337; AVX512-NEXT: vpsllw $15, %xmm1, %xmm1338; AVX512-NEXT: vpmovw2m %xmm1, %k1339; AVX512-NEXT: vmovdqu16 %xmm0, (%rdi) {%k1}340; AVX512-NEXT: retq341 %load = load <8 x i16>, ptr %ptr, align 32342 %sel = select <8 x i1> %mask, <8 x i16> %x, <8 x i16> %load343 store <8 x i16> %sel, ptr %ptr, align 32344 ret void345}346 347define void @test_masked_store_success_v8i32(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {348; AVX-LABEL: test_masked_store_success_v8i32:349; AVX: # %bb.0:350; AVX-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero351; AVX-NEXT: vpslld $31, %xmm2, %xmm2352; AVX-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]353; AVX-NEXT: vpslld $31, %xmm1, %xmm1354; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1355; AVX-NEXT: vmaskmovps %ymm0, %ymm1, (%rdi)356; AVX-NEXT: vzeroupper357; AVX-NEXT: retq358;359; AVX2-LABEL: test_masked_store_success_v8i32:360; AVX2: # %bb.0:361; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero362; AVX2-NEXT: vpslld $31, %ymm1, %ymm1363; AVX2-NEXT: vpmaskmovd %ymm0, %ymm1, (%rdi)364; AVX2-NEXT: vzeroupper365; AVX2-NEXT: retq366;367; AVX512-LABEL: test_masked_store_success_v8i32:368; AVX512: # %bb.0:369; AVX512-NEXT: vpsllw $15, %xmm1, %xmm1370; AVX512-NEXT: vpmovw2m %xmm1, %k1371; AVX512-NEXT: vmovdqa32 %ymm0, (%rdi) {%k1}372; AVX512-NEXT: vzeroupper373; AVX512-NEXT: retq374 %load = load <8 x i32>, ptr %ptr, align 32375 %sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load376 store <8 x i32> %sel, ptr %ptr, align 32377 ret void378}379 380define void @test_masked_store_success_v8i64(<8 x i64> %x, ptr %ptr, <8 x i1> %mask) {381; AVX-LABEL: test_masked_store_success_v8i64:382; AVX: # %bb.0:383; AVX-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7]384; AVX-NEXT: vpslld $31, %xmm3, %xmm3385; AVX-NEXT: vpmovsxdq %xmm3, %xmm4386; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]387; AVX-NEXT: vpmovsxdq %xmm3, %xmm3388; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm3389; AVX-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero390; AVX-NEXT: vpslld $31, %xmm2, %xmm2391; AVX-NEXT: vpmovsxdq %xmm2, %xmm4392; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]393; AVX-NEXT: vpmovsxdq %xmm2, %xmm2394; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm4, %ymm2395; AVX-NEXT: vmaskmovpd %ymm0, %ymm2, (%rdi)396; AVX-NEXT: vmaskmovpd %ymm1, %ymm3, 32(%rdi)397; AVX-NEXT: vzeroupper398; AVX-NEXT: retq399;400; AVX2-LABEL: test_masked_store_success_v8i64:401; AVX2: # %bb.0:402; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7]403; AVX2-NEXT: vpslld $31, %xmm3, %xmm3404; AVX2-NEXT: vpmovsxdq %xmm3, %ymm3405; AVX2-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero406; AVX2-NEXT: vpslld $31, %xmm2, %xmm2407; AVX2-NEXT: vpmovsxdq %xmm2, %ymm2408; AVX2-NEXT: vpmaskmovq %ymm0, %ymm2, (%rdi)409; AVX2-NEXT: vpmaskmovq %ymm1, %ymm3, 32(%rdi)410; AVX2-NEXT: vzeroupper411; AVX2-NEXT: retq412;413; AVX512-LABEL: test_masked_store_success_v8i64:414; AVX512: # %bb.0:415; AVX512-NEXT: vpsllw $15, %xmm1, %xmm1416; AVX512-NEXT: vpmovw2m %xmm1, %k1417; AVX512-NEXT: vmovdqu64 %zmm0, (%rdi) {%k1}418; AVX512-NEXT: vzeroupper419; AVX512-NEXT: retq420 %load = load <8 x i64>, ptr %ptr, align 32421 %sel = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> %load422 store <8 x i64> %sel, ptr %ptr, align 32423 ret void424}425 426define void @test_masked_store_success_v8f16(<8 x half> %x, ptr %ptr, <8 x i1> %mask) {427; AVX-LABEL: test_masked_store_success_v8f16:428; AVX: # %bb.0:429; AVX-NEXT: vpsllw $15, %xmm1, %xmm1430; AVX-NEXT: vpsraw $15, %xmm1, %xmm1431; AVX-NEXT: vmovdqa (%rdi), %xmm2432; AVX-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm0433; AVX-NEXT: vmovdqa %xmm0, (%rdi)434; AVX-NEXT: retq435;436; AVX2-LABEL: test_masked_store_success_v8f16:437; AVX2: # %bb.0:438; AVX2-NEXT: vpsllw $15, %xmm1, %xmm1439; AVX2-NEXT: vpsraw $15, %xmm1, %xmm1440; AVX2-NEXT: vmovdqa (%rdi), %xmm2441; AVX2-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm0442; AVX2-NEXT: vmovdqa %xmm0, (%rdi)443; AVX2-NEXT: retq444;445; AVX512-LABEL: test_masked_store_success_v8f16:446; AVX512: # %bb.0:447; AVX512-NEXT: vpsllw $15, %xmm1, %xmm1448; AVX512-NEXT: vpmovw2m %xmm1, %k1449; AVX512-NEXT: vmovdqu16 %xmm0, (%rdi) {%k1}450; AVX512-NEXT: retq451 %load = load <8 x half>, ptr %ptr, align 32452 %sel = select <8 x i1> %mask, <8 x half> %x, <8 x half> %load453 store <8 x half> %sel, ptr %ptr, align 32454 ret void455}456 457define void @test_masked_store_success_v8f32(<8 x float> %x, ptr %ptr, <8 x i1> %mask) {458; AVX-LABEL: test_masked_store_success_v8f32:459; AVX: # %bb.0:460; AVX-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero461; AVX-NEXT: vpslld $31, %xmm2, %xmm2462; AVX-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]463; AVX-NEXT: vpslld $31, %xmm1, %xmm1464; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1465; AVX-NEXT: vmaskmovps %ymm0, %ymm1, (%rdi)466; AVX-NEXT: vzeroupper467; AVX-NEXT: retq468;469; AVX2-LABEL: test_masked_store_success_v8f32:470; AVX2: # %bb.0:471; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero472; AVX2-NEXT: vpslld $31, %ymm1, %ymm1473; AVX2-NEXT: vmaskmovps %ymm0, %ymm1, (%rdi)474; AVX2-NEXT: vzeroupper475; AVX2-NEXT: retq476;477; AVX512-LABEL: test_masked_store_success_v8f32:478; AVX512: # %bb.0:479; AVX512-NEXT: vpsllw $15, %xmm1, %xmm1480; AVX512-NEXT: vpmovw2m %xmm1, %k1481; AVX512-NEXT: vmovaps %ymm0, (%rdi) {%k1}482; AVX512-NEXT: vzeroupper483; AVX512-NEXT: retq484 %load = load <8 x float>, ptr %ptr, align 32485 %sel = select <8 x i1> %mask, <8 x float> %x, <8 x float> %load486 store <8 x float> %sel, ptr %ptr, align 32487 ret void488}489 490define void @test_masked_store_success_v8f64(<8 x double> %x, ptr %ptr, <8 x i1> %mask) {491; AVX-LABEL: test_masked_store_success_v8f64:492; AVX: # %bb.0:493; AVX-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7]494; AVX-NEXT: vpslld $31, %xmm3, %xmm3495; AVX-NEXT: vpmovsxdq %xmm3, %xmm4496; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]497; AVX-NEXT: vpmovsxdq %xmm3, %xmm3498; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm3499; AVX-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero500; AVX-NEXT: vpslld $31, %xmm2, %xmm2501; AVX-NEXT: vpmovsxdq %xmm2, %xmm4502; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]503; AVX-NEXT: vpmovsxdq %xmm2, %xmm2504; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm4, %ymm2505; AVX-NEXT: vmaskmovpd %ymm0, %ymm2, (%rdi)506; AVX-NEXT: vmaskmovpd %ymm1, %ymm3, 32(%rdi)507; AVX-NEXT: vzeroupper508; AVX-NEXT: retq509;510; AVX2-LABEL: test_masked_store_success_v8f64:511; AVX2: # %bb.0:512; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7]513; AVX2-NEXT: vpslld $31, %xmm3, %xmm3514; AVX2-NEXT: vpmovsxdq %xmm3, %ymm3515; AVX2-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero516; AVX2-NEXT: vpslld $31, %xmm2, %xmm2517; AVX2-NEXT: vpmovsxdq %xmm2, %ymm2518; AVX2-NEXT: vmaskmovpd %ymm0, %ymm2, (%rdi)519; AVX2-NEXT: vmaskmovpd %ymm1, %ymm3, 32(%rdi)520; AVX2-NEXT: vzeroupper521; AVX2-NEXT: retq522;523; AVX512-LABEL: test_masked_store_success_v8f64:524; AVX512: # %bb.0:525; AVX512-NEXT: vpsllw $15, %xmm1, %xmm1526; AVX512-NEXT: vpmovw2m %xmm1, %k1527; AVX512-NEXT: vmovupd %zmm0, (%rdi) {%k1}528; AVX512-NEXT: vzeroupper529; AVX512-NEXT: retq530 %load = load <8 x double>, ptr %ptr, align 32531 %sel = select <8 x i1> %mask, <8 x double> %x, <8 x double> %load532 store <8 x double> %sel, ptr %ptr, align 32533 ret void534}535 536define void @test_masked_store_success_v16i8(<16 x i8> %x, ptr %ptr, <16 x i1> %mask) {537; AVX-LABEL: test_masked_store_success_v16i8:538; AVX: # %bb.0:539; AVX-NEXT: vpsllw $7, %xmm1, %xmm1540; AVX-NEXT: vmovdqa (%rdi), %xmm2541; AVX-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm0542; AVX-NEXT: vmovdqa %xmm0, (%rdi)543; AVX-NEXT: retq544;545; AVX2-LABEL: test_masked_store_success_v16i8:546; AVX2: # %bb.0:547; AVX2-NEXT: vpsllw $7, %xmm1, %xmm1548; AVX2-NEXT: vmovdqa (%rdi), %xmm2549; AVX2-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm0550; AVX2-NEXT: vmovdqa %xmm0, (%rdi)551; AVX2-NEXT: retq552;553; AVX512-LABEL: test_masked_store_success_v16i8:554; AVX512: # %bb.0:555; AVX512-NEXT: vpsllw $7, %xmm1, %xmm1556; AVX512-NEXT: vpmovb2m %xmm1, %k1557; AVX512-NEXT: vmovdqu8 %xmm0, (%rdi) {%k1}558; AVX512-NEXT: retq559 %load = load <16 x i8>, ptr %ptr, align 32560 %sel = select <16 x i1> %mask, <16 x i8> %x, <16 x i8> %load561 store <16 x i8> %sel, ptr %ptr, align 32562 ret void563}564 565define void @test_masked_store_success_v16i16(<16 x i16> %x, ptr %ptr, <16 x i1> %mask) {566; AVX-LABEL: test_masked_store_success_v16i16:567; AVX: # %bb.0:568; AVX-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]569; AVX-NEXT: vpsllw $15, %xmm2, %xmm2570; AVX-NEXT: vpsraw $15, %xmm2, %xmm2571; AVX-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero572; AVX-NEXT: vpsllw $15, %xmm1, %xmm1573; AVX-NEXT: vpsraw $15, %xmm1, %xmm1574; AVX-NEXT: vmovdqa (%rdi), %xmm3575; AVX-NEXT: vpblendvb %xmm1, %xmm0, %xmm3, %xmm1576; AVX-NEXT: vmovdqa 16(%rdi), %xmm3577; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0578; AVX-NEXT: vpblendvb %xmm2, %xmm0, %xmm3, %xmm0579; AVX-NEXT: vmovdqa %xmm1, (%rdi)580; AVX-NEXT: vmovdqa %xmm0, 16(%rdi)581; AVX-NEXT: vzeroupper582; AVX-NEXT: retq583;584; AVX2-LABEL: test_masked_store_success_v16i16:585; AVX2: # %bb.0:586; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero587; AVX2-NEXT: vpsllw $15, %ymm1, %ymm1588; AVX2-NEXT: vpsraw $15, %ymm1, %ymm1589; AVX2-NEXT: vmovdqa (%rdi), %ymm2590; AVX2-NEXT: vpblendvb %ymm1, %ymm0, %ymm2, %ymm0591; AVX2-NEXT: vmovdqa %ymm0, (%rdi)592; AVX2-NEXT: vzeroupper593; AVX2-NEXT: retq594;595; AVX512-LABEL: test_masked_store_success_v16i16:596; AVX512: # %bb.0:597; AVX512-NEXT: vpsllw $7, %xmm1, %xmm1598; AVX512-NEXT: vpmovb2m %xmm1, %k1599; AVX512-NEXT: vmovdqu16 %ymm0, (%rdi) {%k1}600; AVX512-NEXT: vzeroupper601; AVX512-NEXT: retq602 %load = load <16 x i16>, ptr %ptr, align 32603 %sel = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> %load604 store <16 x i16> %sel, ptr %ptr, align 32605 ret void606}607 608define void @test_masked_store_success_v16i32(<16 x i32> %x, ptr %ptr, <16 x i1> %mask) {609; AVX-LABEL: test_masked_store_success_v16i32:610; AVX: # %bb.0:611; AVX-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]612; AVX-NEXT: vpmovzxwd {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero613; AVX-NEXT: vpslld $31, %xmm4, %xmm4614; AVX-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]615; AVX-NEXT: vpslld $31, %xmm3, %xmm3616; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm3617; AVX-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero618; AVX-NEXT: vpslld $31, %xmm4, %xmm4619; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]620; AVX-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero621; AVX-NEXT: vpslld $31, %xmm2, %xmm2622; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm4, %ymm2623; AVX-NEXT: vmaskmovps %ymm0, %ymm2, (%rdi)624; AVX-NEXT: vmaskmovps %ymm1, %ymm3, 32(%rdi)625; AVX-NEXT: vzeroupper626; AVX-NEXT: retq627;628; AVX2-LABEL: test_masked_store_success_v16i32:629; AVX2: # %bb.0:630; AVX2-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]631; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero632; AVX2-NEXT: vpslld $31, %ymm3, %ymm3633; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero634; AVX2-NEXT: vpslld $31, %ymm2, %ymm2635; AVX2-NEXT: vpmaskmovd %ymm0, %ymm2, (%rdi)636; AVX2-NEXT: vpmaskmovd %ymm1, %ymm3, 32(%rdi)637; AVX2-NEXT: vzeroupper638; AVX2-NEXT: retq639;640; AVX512-LABEL: test_masked_store_success_v16i32:641; AVX512: # %bb.0:642; AVX512-NEXT: vpsllw $7, %xmm1, %xmm1643; AVX512-NEXT: vpmovb2m %xmm1, %k1644; AVX512-NEXT: vmovdqu32 %zmm0, (%rdi) {%k1}645; AVX512-NEXT: vzeroupper646; AVX512-NEXT: retq647 %load = load <16 x i32>, ptr %ptr, align 32648 %sel = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %load649 store <16 x i32> %sel, ptr %ptr, align 32650 ret void651}652 653define void @test_masked_store_success_v32i8(<32 x i8> %x, ptr %ptr, <32 x i1> %mask) {654; AVX-LABEL: test_masked_store_success_v32i8:655; AVX: # %bb.0:656; AVX-NEXT: vpsllw $7, %xmm1, %xmm2657; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1658; AVX-NEXT: vpsllw $7, %xmm1, %xmm1659; AVX-NEXT: vextractf128 $1, %ymm0, %xmm3660; AVX-NEXT: vmovdqa (%rdi), %xmm4661; AVX-NEXT: vmovdqa 16(%rdi), %xmm5662; AVX-NEXT: vpblendvb %xmm1, %xmm3, %xmm5, %xmm1663; AVX-NEXT: vpblendvb %xmm2, %xmm0, %xmm4, %xmm0664; AVX-NEXT: vmovdqa %xmm0, (%rdi)665; AVX-NEXT: vmovdqa %xmm1, 16(%rdi)666; AVX-NEXT: vzeroupper667; AVX-NEXT: retq668;669; AVX2-LABEL: test_masked_store_success_v32i8:670; AVX2: # %bb.0:671; AVX2-NEXT: vpsllw $7, %ymm1, %ymm1672; AVX2-NEXT: vmovdqa (%rdi), %ymm2673; AVX2-NEXT: vpblendvb %ymm1, %ymm0, %ymm2, %ymm0674; AVX2-NEXT: vmovdqa %ymm0, (%rdi)675; AVX2-NEXT: vzeroupper676; AVX2-NEXT: retq677;678; AVX512-LABEL: test_masked_store_success_v32i8:679; AVX512: # %bb.0:680; AVX512-NEXT: vpsllw $7, %ymm1, %ymm1681; AVX512-NEXT: vpmovb2m %ymm1, %k1682; AVX512-NEXT: vmovdqu8 %ymm0, (%rdi) {%k1}683; AVX512-NEXT: vzeroupper684; AVX512-NEXT: retq685 %load = load <32 x i8>, ptr %ptr, align 32686 %sel = select <32 x i1> %mask, <32 x i8> %x, <32 x i8> %load687 store <32 x i8> %sel, ptr %ptr, align 32688 ret void689}690 691define void @test_masked_store_success_v32i16(<32 x i16> %x, ptr %ptr, <32 x i1> %mask) {692; AVX-LABEL: test_masked_store_success_v32i16:693; AVX: # %bb.0:694; AVX-NEXT: vmovdqa (%rdi), %xmm3695; AVX-NEXT: vmovdqa 16(%rdi), %xmm4696; AVX-NEXT: vmovdqa 32(%rdi), %xmm5697; AVX-NEXT: vmovdqa 48(%rdi), %xmm6698; AVX-NEXT: vextractf128 $1, %ymm2, %xmm7699; AVX-NEXT: vpmovzxbw {{.*#+}} xmm8 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero700; AVX-NEXT: vpsllw $15, %xmm8, %xmm8701; AVX-NEXT: vpsraw $15, %xmm8, %xmm8702; AVX-NEXT: vpblendvb %xmm8, %xmm1, %xmm5, %xmm5703; AVX-NEXT: vpmovzxbw {{.*#+}} xmm8 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero704; AVX-NEXT: vpsllw $15, %xmm8, %xmm8705; AVX-NEXT: vpsraw $15, %xmm8, %xmm8706; AVX-NEXT: vpblendvb %xmm8, %xmm0, %xmm3, %xmm3707; AVX-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]708; AVX-NEXT: vpsllw $15, %xmm7, %xmm7709; AVX-NEXT: vpsraw $15, %xmm7, %xmm7710; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1711; AVX-NEXT: vpblendvb %xmm7, %xmm1, %xmm6, %xmm1712; AVX-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]713; AVX-NEXT: vpsllw $15, %xmm2, %xmm2714; AVX-NEXT: vpsraw $15, %xmm2, %xmm2715; AVX-NEXT: vextractf128 $1, %ymm0, %xmm0716; AVX-NEXT: vpblendvb %xmm2, %xmm0, %xmm4, %xmm0717; AVX-NEXT: vmovdqa %xmm3, (%rdi)718; AVX-NEXT: vmovdqa %xmm0, 16(%rdi)719; AVX-NEXT: vmovdqa %xmm5, 32(%rdi)720; AVX-NEXT: vmovdqa %xmm1, 48(%rdi)721; AVX-NEXT: vzeroupper722; AVX-NEXT: retq723;724; AVX2-LABEL: test_masked_store_success_v32i16:725; AVX2: # %bb.0:726; AVX2-NEXT: vmovdqa (%rdi), %ymm3727; AVX2-NEXT: vmovdqa 32(%rdi), %ymm4728; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm5729; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero730; AVX2-NEXT: vpsllw $15, %ymm5, %ymm5731; AVX2-NEXT: vpsraw $15, %ymm5, %ymm5732; AVX2-NEXT: vpblendvb %ymm5, %ymm1, %ymm4, %ymm1733; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero734; AVX2-NEXT: vpsllw $15, %ymm2, %ymm2735; AVX2-NEXT: vpsraw $15, %ymm2, %ymm2736; AVX2-NEXT: vpblendvb %ymm2, %ymm0, %ymm3, %ymm0737; AVX2-NEXT: vmovdqa %ymm0, (%rdi)738; AVX2-NEXT: vmovdqa %ymm1, 32(%rdi)739; AVX2-NEXT: vzeroupper740; AVX2-NEXT: retq741;742; AVX512-LABEL: test_masked_store_success_v32i16:743; AVX512: # %bb.0:744; AVX512-NEXT: vpsllw $7, %ymm1, %ymm1745; AVX512-NEXT: vpmovb2m %ymm1, %k1746; AVX512-NEXT: vmovdqu16 %zmm0, (%rdi) {%k1}747; AVX512-NEXT: vzeroupper748; AVX512-NEXT: retq749 %load = load <32 x i16>, ptr %ptr, align 32750 %sel = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> %load751 store <32 x i16> %sel, ptr %ptr, align 32752 ret void753}754 755define void @test_masked_store_success_v64i8(<64 x i8> %x, ptr %ptr, <64 x i1> %mask) {756; AVX-LABEL: test_masked_store_success_v64i8:757; AVX: # %bb.0:758; AVX-NEXT: vmovd %esi, %xmm2759; AVX-NEXT: vpinsrb $1, %edx, %xmm2, %xmm2760; AVX-NEXT: vpinsrb $2, %ecx, %xmm2, %xmm2761; AVX-NEXT: vpinsrb $3, %r8d, %xmm2, %xmm2762; AVX-NEXT: vpinsrb $4, %r9d, %xmm2, %xmm2763; AVX-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2764; AVX-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2765; AVX-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2766; AVX-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2767; AVX-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2768; AVX-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2769; AVX-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2770; AVX-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2771; AVX-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2772; AVX-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2773; AVX-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2774; AVX-NEXT: vpsllw $7, %xmm2, %xmm2775; AVX-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero776; AVX-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm3, %xmm3777; AVX-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm3, %xmm3778; AVX-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm3, %xmm3779; AVX-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm3, %xmm3780; AVX-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm3, %xmm3781; AVX-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm3, %xmm3782; AVX-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm3, %xmm3783; AVX-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm3, %xmm3784; AVX-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm3, %xmm3785; AVX-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm3, %xmm3786; AVX-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm3, %xmm3787; AVX-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm3, %xmm3788; AVX-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm3, %xmm3789; AVX-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm3, %xmm3790; AVX-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm3, %xmm3791; AVX-NEXT: vpsllw $7, %xmm3, %xmm3792; AVX-NEXT: vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero793; AVX-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm4, %xmm4794; AVX-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm4, %xmm4795; AVX-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm4, %xmm4796; AVX-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm4, %xmm4797; AVX-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm4, %xmm4798; AVX-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm4, %xmm4799; AVX-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm4, %xmm4800; AVX-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm4, %xmm4801; AVX-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm4, %xmm4802; AVX-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm4, %xmm4803; AVX-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm4, %xmm4804; AVX-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm4, %xmm4805; AVX-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm4, %xmm4806; AVX-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm4, %xmm4807; AVX-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm4, %xmm4808; AVX-NEXT: vpsllw $7, %xmm4, %xmm4809; AVX-NEXT: vmovd {{.*#+}} xmm5 = mem[0],zero,zero,zero810; AVX-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm5, %xmm5811; AVX-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm5, %xmm5812; AVX-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm5, %xmm5813; AVX-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm5, %xmm5814; AVX-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm5, %xmm5815; AVX-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm5, %xmm5816; AVX-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm5, %xmm5817; AVX-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm5, %xmm5818; AVX-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm5, %xmm5819; AVX-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm5, %xmm5820; AVX-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm5, %xmm5821; AVX-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm5, %xmm5822; AVX-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm5, %xmm5823; AVX-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm5, %xmm5824; AVX-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm5, %xmm5825; AVX-NEXT: vpsllw $7, %xmm5, %xmm5826; AVX-NEXT: vextractf128 $1, %ymm1, %xmm6827; AVX-NEXT: vmovdqa (%rdi), %xmm7828; AVX-NEXT: vmovdqa 16(%rdi), %xmm8829; AVX-NEXT: vmovdqa 32(%rdi), %xmm9830; AVX-NEXT: vmovdqa 48(%rdi), %xmm10831; AVX-NEXT: vpblendvb %xmm5, %xmm6, %xmm10, %xmm5832; AVX-NEXT: vpblendvb %xmm4, %xmm1, %xmm9, %xmm1833; AVX-NEXT: vextractf128 $1, %ymm0, %xmm4834; AVX-NEXT: vpblendvb %xmm3, %xmm4, %xmm8, %xmm3835; AVX-NEXT: vpblendvb %xmm2, %xmm0, %xmm7, %xmm0836; AVX-NEXT: vmovdqa %xmm3, 16(%rdi)837; AVX-NEXT: vmovdqa %xmm1, 32(%rdi)838; AVX-NEXT: vmovdqa %xmm5, 48(%rdi)839; AVX-NEXT: vmovdqa %xmm0, (%rdi)840; AVX-NEXT: vzeroupper841; AVX-NEXT: retq842;843; AVX2-LABEL: test_masked_store_success_v64i8:844; AVX2: # %bb.0:845; AVX2-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero846; AVX2-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2847; AVX2-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2848; AVX2-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2849; AVX2-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm2, %xmm2850; AVX2-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm2, %xmm2851; AVX2-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm2, %xmm2852; AVX2-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm2, %xmm2853; AVX2-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm2, %xmm2854; AVX2-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm2, %xmm2855; AVX2-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm2, %xmm2856; AVX2-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm2, %xmm2857; AVX2-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm2, %xmm2858; AVX2-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm2, %xmm2859; AVX2-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm2, %xmm2860; AVX2-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm2, %xmm2861; AVX2-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero862; AVX2-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm3, %xmm3863; AVX2-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm3, %xmm3864; AVX2-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm3, %xmm3865; AVX2-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm3, %xmm3866; AVX2-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm3, %xmm3867; AVX2-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm3, %xmm3868; AVX2-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm3, %xmm3869; AVX2-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm3, %xmm3870; AVX2-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm3, %xmm3871; AVX2-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm3, %xmm3872; AVX2-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm3, %xmm3873; AVX2-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm3, %xmm3874; AVX2-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm3, %xmm3875; AVX2-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm3, %xmm3876; AVX2-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm3, %xmm3877; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2878; AVX2-NEXT: vpsllw $7, %ymm2, %ymm2879; AVX2-NEXT: vmovd %esi, %xmm3880; AVX2-NEXT: vpinsrb $1, %edx, %xmm3, %xmm3881; AVX2-NEXT: vpinsrb $2, %ecx, %xmm3, %xmm3882; AVX2-NEXT: vpinsrb $3, %r8d, %xmm3, %xmm3883; AVX2-NEXT: vpinsrb $4, %r9d, %xmm3, %xmm3884; AVX2-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm3, %xmm3885; AVX2-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm3, %xmm3886; AVX2-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm3, %xmm3887; AVX2-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm3, %xmm3888; AVX2-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm3, %xmm3889; AVX2-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm3, %xmm3890; AVX2-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm3, %xmm3891; AVX2-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm3, %xmm3892; AVX2-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm3, %xmm3893; AVX2-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm3, %xmm3894; AVX2-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm3, %xmm3895; AVX2-NEXT: vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero896; AVX2-NEXT: vpinsrb $1, {{[0-9]+}}(%rsp), %xmm4, %xmm4897; AVX2-NEXT: vpinsrb $2, {{[0-9]+}}(%rsp), %xmm4, %xmm4898; AVX2-NEXT: vpinsrb $3, {{[0-9]+}}(%rsp), %xmm4, %xmm4899; AVX2-NEXT: vpinsrb $4, {{[0-9]+}}(%rsp), %xmm4, %xmm4900; AVX2-NEXT: vpinsrb $5, {{[0-9]+}}(%rsp), %xmm4, %xmm4901; AVX2-NEXT: vpinsrb $6, {{[0-9]+}}(%rsp), %xmm4, %xmm4902; AVX2-NEXT: vpinsrb $7, {{[0-9]+}}(%rsp), %xmm4, %xmm4903; AVX2-NEXT: vpinsrb $8, {{[0-9]+}}(%rsp), %xmm4, %xmm4904; AVX2-NEXT: vpinsrb $9, {{[0-9]+}}(%rsp), %xmm4, %xmm4905; AVX2-NEXT: vpinsrb $10, {{[0-9]+}}(%rsp), %xmm4, %xmm4906; AVX2-NEXT: vpinsrb $11, {{[0-9]+}}(%rsp), %xmm4, %xmm4907; AVX2-NEXT: vpinsrb $12, {{[0-9]+}}(%rsp), %xmm4, %xmm4908; AVX2-NEXT: vpinsrb $13, {{[0-9]+}}(%rsp), %xmm4, %xmm4909; AVX2-NEXT: vpinsrb $14, {{[0-9]+}}(%rsp), %xmm4, %xmm4910; AVX2-NEXT: vpinsrb $15, {{[0-9]+}}(%rsp), %xmm4, %xmm4911; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3912; AVX2-NEXT: vpsllw $7, %ymm3, %ymm3913; AVX2-NEXT: vmovdqa (%rdi), %ymm4914; AVX2-NEXT: vpblendvb %ymm3, %ymm0, %ymm4, %ymm0915; AVX2-NEXT: vmovdqa 32(%rdi), %ymm3916; AVX2-NEXT: vpblendvb %ymm2, %ymm1, %ymm3, %ymm1917; AVX2-NEXT: vmovdqa %ymm1, 32(%rdi)918; AVX2-NEXT: vmovdqa %ymm0, (%rdi)919; AVX2-NEXT: vzeroupper920; AVX2-NEXT: retq921;922; AVX512-LABEL: test_masked_store_success_v64i8:923; AVX512: # %bb.0:924; AVX512-NEXT: vpsllw $7, %zmm1, %zmm1925; AVX512-NEXT: vpmovb2m %zmm1, %k1926; AVX512-NEXT: vmovdqu8 %zmm0, (%rdi) {%k1}927; AVX512-NEXT: vzeroupper928; AVX512-NEXT: retq929 %load = load <64 x i8>, ptr %ptr, align 32930 %sel = select <64 x i1> %mask, <64 x i8> %x, <64 x i8> %load931 store <64 x i8> %sel, ptr %ptr, align 32932 ret void933}934 935define void @test_masked_store_success_invert_mask_v4i32(<4 x i32> %x, ptr %ptr, <4 x i1> %mask) {936; AVX-LABEL: test_masked_store_success_invert_mask_v4i32:937; AVX: # %bb.0:938; AVX-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2939; AVX-NEXT: vpxor %xmm2, %xmm1, %xmm1940; AVX-NEXT: vpslld $31, %xmm1, %xmm1941; AVX-NEXT: vmaskmovps %xmm0, %xmm1, (%rdi)942; AVX-NEXT: retq943;944; AVX2-LABEL: test_masked_store_success_invert_mask_v4i32:945; AVX2: # %bb.0:946; AVX2-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2947; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm1948; AVX2-NEXT: vpslld $31, %xmm1, %xmm1949; AVX2-NEXT: vpmaskmovd %xmm0, %xmm1, (%rdi)950; AVX2-NEXT: retq951;952; AVX512-LABEL: test_masked_store_success_invert_mask_v4i32:953; AVX512: # %bb.0:954; AVX512-NEXT: vpslld $31, %xmm1, %xmm1955; AVX512-NEXT: vpmovd2m %xmm1, %k0956; AVX512-NEXT: knotw %k0, %k1957; AVX512-NEXT: vmovdqa32 %xmm0, (%rdi) {%k1}958; AVX512-NEXT: retq959 %load = load <4 x i32>, ptr %ptr, align 32960 %sel = select <4 x i1> %mask, <4 x i32> %load, <4 x i32> %x961 store <4 x i32> %sel, ptr %ptr, align 32962 ret void963}964 965define void @test_masked_store_success_invert_mask_v8i32(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {966; AVX-LABEL: test_masked_store_success_invert_mask_v8i32:967; AVX: # %bb.0:968; AVX-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2969; AVX-NEXT: vpxor %xmm2, %xmm1, %xmm1970; AVX-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero971; AVX-NEXT: vpslld $31, %xmm2, %xmm2972; AVX-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]973; AVX-NEXT: vpslld $31, %xmm1, %xmm1974; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1975; AVX-NEXT: vmaskmovps %ymm0, %ymm1, (%rdi)976; AVX-NEXT: vzeroupper977; AVX-NEXT: retq978;979; AVX2-LABEL: test_masked_store_success_invert_mask_v8i32:980; AVX2: # %bb.0:981; AVX2-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2982; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm1983; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero984; AVX2-NEXT: vpslld $31, %ymm1, %ymm1985; AVX2-NEXT: vpmaskmovd %ymm0, %ymm1, (%rdi)986; AVX2-NEXT: vzeroupper987; AVX2-NEXT: retq988;989; AVX512-LABEL: test_masked_store_success_invert_mask_v8i32:990; AVX512: # %bb.0:991; AVX512-NEXT: vpsllw $15, %xmm1, %xmm1992; AVX512-NEXT: vpmovw2m %xmm1, %k0993; AVX512-NEXT: knotb %k0, %k1994; AVX512-NEXT: vmovdqa32 %ymm0, (%rdi) {%k1}995; AVX512-NEXT: vzeroupper996; AVX512-NEXT: retq997 %load = load <8 x i32>, ptr %ptr, align 32998 %sel = select <8 x i1> %mask, <8 x i32> %load, <8 x i32> %x999 store <8 x i32> %sel, ptr %ptr, align 321000 ret void1001}1002 1003define void @test_masked_store_success_invert_mask_v16i32(<16 x i32> %x, ptr %ptr, <16 x i1> %mask) {1004; AVX-LABEL: test_masked_store_success_invert_mask_v16i32:1005; AVX: # %bb.0:1006; AVX-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero1007; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]1008; AVX-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero1009; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm31010; AVX-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1011; AVX-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm2[4,4,5,5,6,6,7,7]1012; AVX-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero1013; AVX-NEXT: vpcmpeqd %xmm5, %xmm5, %xmm51014; AVX-NEXT: vpxor %xmm5, %xmm2, %xmm21015; AVX-NEXT: vpslld $31, %xmm2, %xmm21016; AVX-NEXT: vpxor %xmm5, %xmm4, %xmm41017; AVX-NEXT: vpslld $31, %xmm4, %xmm41018; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm2, %ymm21019; AVX-NEXT: vmaskmovps %ymm1, %ymm2, 32(%rdi)1020; AVX-NEXT: vxorps %xmm1, %xmm1, %xmm11021; AVX-NEXT: vcmptrueps %ymm1, %ymm1, %ymm11022; AVX-NEXT: vxorps %ymm1, %ymm3, %ymm11023; AVX-NEXT: vpslld $31, %xmm1, %xmm21024; AVX-NEXT: vextractf128 $1, %ymm1, %xmm11025; AVX-NEXT: vpslld $31, %xmm1, %xmm11026; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm11027; AVX-NEXT: vmaskmovps %ymm0, %ymm1, (%rdi)1028; AVX-NEXT: vzeroupper1029; AVX-NEXT: retq1030;1031; AVX2-LABEL: test_masked_store_success_invert_mask_v16i32:1032; AVX2: # %bb.0:1033; AVX2-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1034; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero1035; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero1036; AVX2-NEXT: vpcmpeqd %ymm4, %ymm4, %ymm41037; AVX2-NEXT: vpxor %ymm4, %ymm2, %ymm21038; AVX2-NEXT: vpslld $31, %ymm2, %ymm21039; AVX2-NEXT: vpmaskmovd %ymm0, %ymm2, (%rdi)1040; AVX2-NEXT: vpxor %ymm4, %ymm3, %ymm01041; AVX2-NEXT: vpslld $31, %ymm0, %ymm01042; AVX2-NEXT: vpmaskmovd %ymm1, %ymm0, 32(%rdi)1043; AVX2-NEXT: vzeroupper1044; AVX2-NEXT: retq1045;1046; AVX512-LABEL: test_masked_store_success_invert_mask_v16i32:1047; AVX512: # %bb.0:1048; AVX512-NEXT: vpsllw $7, %xmm1, %xmm11049; AVX512-NEXT: vpmovb2m %xmm1, %k01050; AVX512-NEXT: knotw %k0, %k11051; AVX512-NEXT: vmovdqu32 %zmm0, (%rdi) {%k1}1052; AVX512-NEXT: vzeroupper1053; AVX512-NEXT: retq1054 %load = load <16 x i32>, ptr %ptr, align 321055 %sel = select <16 x i1> %mask, <16 x i32> %load, <16 x i32> %x1056 store <16 x i32> %sel, ptr %ptr, align 321057 ret void1058}1059 1060define void @test_masked_store_zextload(<4 x i64> %x, ptr %ptr, <4 x i1> %mask) {1061; AVX-LABEL: test_masked_store_zextload:1062; AVX: # %bb.0:1063; AVX-NEXT: vpslld $31, %xmm1, %xmm11064; AVX-NEXT: vpmovsxdq %xmm1, %xmm21065; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]1066; AVX-NEXT: vpmovsxdq %xmm1, %xmm11067; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm11068; AVX-NEXT: vpmovzxdq {{.*#+}} xmm2 = mem[0],zero,mem[1],zero1069; AVX-NEXT: vpmovzxdq {{.*#+}} xmm3 = mem[0],zero,mem[1],zero1070; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm21071; AVX-NEXT: vblendvpd %ymm1, %ymm0, %ymm2, %ymm01072; AVX-NEXT: vmovapd %ymm0, (%rdi)1073; AVX-NEXT: vzeroupper1074; AVX-NEXT: retq1075;1076; AVX2-LABEL: test_masked_store_zextload:1077; AVX2: # %bb.0:1078; AVX2-NEXT: vpslld $31, %xmm1, %xmm11079; AVX2-NEXT: vpmovsxdq %xmm1, %ymm11080; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero1081; AVX2-NEXT: vblendvpd %ymm1, %ymm0, %ymm2, %ymm01082; AVX2-NEXT: vmovapd %ymm0, (%rdi)1083; AVX2-NEXT: vzeroupper1084; AVX2-NEXT: retq1085;1086; AVX512-LABEL: test_masked_store_zextload:1087; AVX512: # %bb.0:1088; AVX512-NEXT: vpslld $31, %xmm1, %xmm11089; AVX512-NEXT: vpmovd2m %xmm1, %k11090; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero1091; AVX512-NEXT: vmovdqa64 %ymm0, %ymm1 {%k1}1092; AVX512-NEXT: vmovdqa %ymm1, (%rdi)1093; AVX512-NEXT: vzeroupper1094; AVX512-NEXT: retq1095 %load = load <4 x i32>, ptr %ptr, align 321096 %zext = zext <4 x i32> %load to <4 x i64>1097 %masked = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> %zext1098 store <4 x i64> %masked, ptr %ptr, align 321099 ret void1100}1101 1102define void @test_masked_store_volatile_load(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {1103; AVX-LABEL: test_masked_store_volatile_load:1104; AVX: # %bb.0:1105; AVX-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero1106; AVX-NEXT: vpslld $31, %xmm2, %xmm21107; AVX-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]1108; AVX-NEXT: vpslld $31, %xmm1, %xmm11109; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm11110; AVX-NEXT: vmovaps (%rdi), %ymm21111; AVX-NEXT: vblendvps %ymm1, %ymm0, %ymm2, %ymm01112; AVX-NEXT: vmovaps %ymm0, (%rdi)1113; AVX-NEXT: vzeroupper1114; AVX-NEXT: retq1115;1116; AVX2-LABEL: test_masked_store_volatile_load:1117; AVX2: # %bb.0:1118; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1119; AVX2-NEXT: vpslld $31, %ymm1, %ymm11120; AVX2-NEXT: vmovaps (%rdi), %ymm21121; AVX2-NEXT: vblendvps %ymm1, %ymm0, %ymm2, %ymm01122; AVX2-NEXT: vmovaps %ymm0, (%rdi)1123; AVX2-NEXT: vzeroupper1124; AVX2-NEXT: retq1125;1126; AVX512-LABEL: test_masked_store_volatile_load:1127; AVX512: # %bb.0:1128; AVX512-NEXT: vpsllw $15, %xmm1, %xmm11129; AVX512-NEXT: vpmovw2m %xmm1, %k11130; AVX512-NEXT: vmovdqa (%rdi), %ymm11131; AVX512-NEXT: vmovdqa32 %ymm0, %ymm1 {%k1}1132; AVX512-NEXT: vmovdqa %ymm1, (%rdi)1133; AVX512-NEXT: vzeroupper1134; AVX512-NEXT: retq1135 %load = load volatile <8 x i32>, ptr %ptr, align 321136 %sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load1137 store <8 x i32> %sel, ptr %ptr, align 321138 ret void1139}1140 1141define void @test_masked_store_volatile_store(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {1142; AVX-LABEL: test_masked_store_volatile_store:1143; AVX: # %bb.0:1144; AVX-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero1145; AVX-NEXT: vpslld $31, %xmm2, %xmm21146; AVX-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]1147; AVX-NEXT: vpslld $31, %xmm1, %xmm11148; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm11149; AVX-NEXT: vmovaps (%rdi), %ymm21150; AVX-NEXT: vblendvps %ymm1, %ymm0, %ymm2, %ymm01151; AVX-NEXT: vmovaps %ymm0, (%rdi)1152; AVX-NEXT: vzeroupper1153; AVX-NEXT: retq1154;1155; AVX2-LABEL: test_masked_store_volatile_store:1156; AVX2: # %bb.0:1157; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1158; AVX2-NEXT: vpslld $31, %ymm1, %ymm11159; AVX2-NEXT: vmovaps (%rdi), %ymm21160; AVX2-NEXT: vblendvps %ymm1, %ymm0, %ymm2, %ymm01161; AVX2-NEXT: vmovaps %ymm0, (%rdi)1162; AVX2-NEXT: vzeroupper1163; AVX2-NEXT: retq1164;1165; AVX512-LABEL: test_masked_store_volatile_store:1166; AVX512: # %bb.0:1167; AVX512-NEXT: vpsllw $15, %xmm1, %xmm11168; AVX512-NEXT: vpmovw2m %xmm1, %k11169; AVX512-NEXT: vmovdqa (%rdi), %ymm11170; AVX512-NEXT: vmovdqa32 %ymm0, %ymm1 {%k1}1171; AVX512-NEXT: vmovdqa %ymm1, (%rdi)1172; AVX512-NEXT: vzeroupper1173; AVX512-NEXT: retq1174 %load = load <8 x i32>, ptr %ptr, align 321175 %sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load1176 store volatile <8 x i32> %sel, ptr %ptr, align 321177 ret void1178}1179 1180declare void @use_vec(<8 x i32>)1181 1182define void @test_masked_store_intervening(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) nounwind {1183; AVX-LABEL: test_masked_store_intervening:1184; AVX: # %bb.0:1185; AVX-NEXT: pushq %rbx1186; AVX-NEXT: subq $32, %rsp1187; AVX-NEXT: movq %rdi, %rbx1188; AVX-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero1189; AVX-NEXT: vpslld $31, %xmm2, %xmm21190; AVX-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]1191; AVX-NEXT: vpslld $31, %xmm1, %xmm11192; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm11193; AVX-NEXT: vmovaps (%rdi), %ymm21194; AVX-NEXT: vblendvps %ymm1, %ymm0, %ymm2, %ymm01195; AVX-NEXT: vmovups %ymm0, (%rsp) # 32-byte Spill1196; AVX-NEXT: vxorps %xmm0, %xmm0, %xmm01197; AVX-NEXT: vmovaps %ymm0, (%rdi)1198; AVX-NEXT: callq use_vec@PLT1199; AVX-NEXT: vmovups (%rsp), %ymm0 # 32-byte Reload1200; AVX-NEXT: vmovaps %ymm0, (%rbx)1201; AVX-NEXT: addq $32, %rsp1202; AVX-NEXT: popq %rbx1203; AVX-NEXT: vzeroupper1204; AVX-NEXT: retq1205;1206; AVX2-LABEL: test_masked_store_intervening:1207; AVX2: # %bb.0:1208; AVX2-NEXT: pushq %rbx1209; AVX2-NEXT: subq $32, %rsp1210; AVX2-NEXT: movq %rdi, %rbx1211; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1212; AVX2-NEXT: vpslld $31, %ymm1, %ymm11213; AVX2-NEXT: vmovaps (%rdi), %ymm21214; AVX2-NEXT: vblendvps %ymm1, %ymm0, %ymm2, %ymm01215; AVX2-NEXT: vmovups %ymm0, (%rsp) # 32-byte Spill1216; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm01217; AVX2-NEXT: vmovaps %ymm0, (%rdi)1218; AVX2-NEXT: callq use_vec@PLT1219; AVX2-NEXT: vmovups (%rsp), %ymm0 # 32-byte Reload1220; AVX2-NEXT: vmovaps %ymm0, (%rbx)1221; AVX2-NEXT: addq $32, %rsp1222; AVX2-NEXT: popq %rbx1223; AVX2-NEXT: vzeroupper1224; AVX2-NEXT: retq1225;1226; AVX512-LABEL: test_masked_store_intervening:1227; AVX512: # %bb.0:1228; AVX512-NEXT: pushq %rbx1229; AVX512-NEXT: subq $80, %rsp1230; AVX512-NEXT: movq %rdi, %rbx1231; AVX512-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1232; AVX512-NEXT: vpsllw $15, %xmm1, %xmm01233; AVX512-NEXT: vpmovw2m %xmm0, %k11234; AVX512-NEXT: kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill1235; AVX512-NEXT: vmovaps (%rdi), %ymm01236; AVX512-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1237; AVX512-NEXT: vxorps %xmm0, %xmm0, %xmm01238; AVX512-NEXT: vmovaps %ymm0, (%rdi)1239; AVX512-NEXT: callq use_vec@PLT1240; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload1241; AVX512-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload1242; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload1243; AVX512-NEXT: vmovdqa32 %ymm0, %ymm1 {%k1}1244; AVX512-NEXT: vmovdqa %ymm1, (%rbx)1245; AVX512-NEXT: addq $80, %rsp1246; AVX512-NEXT: popq %rbx1247; AVX512-NEXT: vzeroupper1248; AVX512-NEXT: retq1249 %load = load <8 x i32>, ptr %ptr, align 321250 store <8 x i32> zeroinitializer, ptr %ptr, align 321251 %tmp = load <8 x i32>, ptr %ptr1252 call void @use_vec(<8 x i32> %tmp)1253 %sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load1254 store <8 x i32> %sel, ptr %ptr, align 321255 ret void1256}1257 1258 1259define void @test_masked_store_multiple_v8i32(<8 x i32> %x, <8 x i32> %y, ptr %ptr1, ptr %ptr2, <8 x i1> %mask, <8 x i1> %mask2) {1260; AVX-LABEL: test_masked_store_multiple_v8i32:1261; AVX: # %bb.0:1262; AVX-NEXT: vpmovzxwd {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero1263; AVX-NEXT: vpslld $31, %xmm4, %xmm41264; AVX-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]1265; AVX-NEXT: vpslld $31, %xmm2, %xmm21266; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm4, %ymm21267; AVX-NEXT: vpmovzxwd {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero1268; AVX-NEXT: vpslld $31, %xmm4, %xmm41269; AVX-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]1270; AVX-NEXT: vpslld $31, %xmm3, %xmm31271; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm31272; AVX-NEXT: vmovaps (%rsi), %ymm41273; AVX-NEXT: vblendvps %ymm3, %ymm1, %ymm4, %ymm11274; AVX-NEXT: vmaskmovps %ymm0, %ymm2, (%rdi)1275; AVX-NEXT: vmovaps %ymm1, (%rsi)1276; AVX-NEXT: vzeroupper1277; AVX-NEXT: retq1278;1279; AVX2-LABEL: test_masked_store_multiple_v8i32:1280; AVX2: # %bb.0:1281; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero1282; AVX2-NEXT: vpslld $31, %ymm2, %ymm21283; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero1284; AVX2-NEXT: vpslld $31, %ymm3, %ymm31285; AVX2-NEXT: vmovaps (%rsi), %ymm41286; AVX2-NEXT: vblendvps %ymm3, %ymm1, %ymm4, %ymm11287; AVX2-NEXT: vpmaskmovd %ymm0, %ymm2, (%rdi)1288; AVX2-NEXT: vmovaps %ymm1, (%rsi)1289; AVX2-NEXT: vzeroupper1290; AVX2-NEXT: retq1291;1292; AVX512-LABEL: test_masked_store_multiple_v8i32:1293; AVX512: # %bb.0:1294; AVX512-NEXT: vpsllw $15, %xmm2, %xmm21295; AVX512-NEXT: vpmovw2m %xmm2, %k11296; AVX512-NEXT: vpsllw $15, %xmm3, %xmm21297; AVX512-NEXT: vmovdqa (%rsi), %ymm31298; AVX512-NEXT: vpmovw2m %xmm2, %k21299; AVX512-NEXT: vmovdqa32 %ymm1, %ymm3 {%k2}1300; AVX512-NEXT: vmovdqa32 %ymm0, (%rdi) {%k1}1301; AVX512-NEXT: vmovdqa %ymm3, (%rsi)1302; AVX512-NEXT: vzeroupper1303; AVX512-NEXT: retq1304 %load = load <8 x i32>, ptr %ptr1, align 321305 %load2 = load <8 x i32>, ptr %ptr2, align 321306 %sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load1307 %sel2 = select <8 x i1> %mask2, <8 x i32> %y, <8 x i32> %load21308 store <8 x i32> %sel, ptr %ptr1, align 321309 store <8 x i32> %sel2, ptr %ptr2, align 321310 ret void1311}1312 1313define void @test_masked_store_multiple_v8i64(<8 x i64> %x, <8 x i64> %y, ptr %ptr1, ptr %ptr2, <8 x i1> %mask, <8 x i1> %mask2) {1314; AVX-LABEL: test_masked_store_multiple_v8i64:1315; AVX: # %bb.0:1316; AVX-NEXT: vmovapd (%rsi), %ymm61317; AVX-NEXT: vmovapd 32(%rsi), %ymm71318; AVX-NEXT: vpunpckhwd {{.*#+}} xmm8 = xmm4[4,4,5,5,6,6,7,7]1319; AVX-NEXT: vpslld $31, %xmm8, %xmm81320; AVX-NEXT: vpmovsxdq %xmm8, %xmm91321; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[2,3,2,3]1322; AVX-NEXT: vpmovsxdq %xmm8, %xmm81323; AVX-NEXT: vinsertf128 $1, %xmm8, %ymm9, %ymm81324; AVX-NEXT: vpmovzxwd {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero1325; AVX-NEXT: vpslld $31, %xmm4, %xmm41326; AVX-NEXT: vpmovsxdq %xmm4, %xmm91327; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]1328; AVX-NEXT: vpmovsxdq %xmm4, %xmm41329; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm9, %ymm41330; AVX-NEXT: vpunpckhwd {{.*#+}} xmm9 = xmm5[4,4,5,5,6,6,7,7]1331; AVX-NEXT: vpslld $31, %xmm9, %xmm91332; AVX-NEXT: vpmovsxdq %xmm9, %xmm101333; AVX-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[2,3,2,3]1334; AVX-NEXT: vpmovsxdq %xmm9, %xmm91335; AVX-NEXT: vinsertf128 $1, %xmm9, %ymm10, %ymm91336; AVX-NEXT: vblendvpd %ymm9, %ymm3, %ymm7, %ymm31337; AVX-NEXT: vpmovzxwd {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero1338; AVX-NEXT: vpslld $31, %xmm5, %xmm51339; AVX-NEXT: vpmovsxdq %xmm5, %xmm71340; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]1341; AVX-NEXT: vpmovsxdq %xmm5, %xmm51342; AVX-NEXT: vinsertf128 $1, %xmm5, %ymm7, %ymm51343; AVX-NEXT: vblendvpd %ymm5, %ymm2, %ymm6, %ymm21344; AVX-NEXT: vmaskmovpd %ymm0, %ymm4, (%rdi)1345; AVX-NEXT: vmaskmovpd %ymm1, %ymm8, 32(%rdi)1346; AVX-NEXT: vmovapd %ymm3, 32(%rsi)1347; AVX-NEXT: vmovapd %ymm2, (%rsi)1348; AVX-NEXT: vzeroupper1349; AVX-NEXT: retq1350;1351; AVX2-LABEL: test_masked_store_multiple_v8i64:1352; AVX2: # %bb.0:1353; AVX2-NEXT: vmovapd (%rsi), %ymm61354; AVX2-NEXT: vmovapd 32(%rsi), %ymm71355; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm8 = xmm4[4,4,5,5,6,6,7,7]1356; AVX2-NEXT: vpslld $31, %xmm8, %xmm81357; AVX2-NEXT: vpmovsxdq %xmm8, %ymm81358; AVX2-NEXT: vpmovzxwd {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero1359; AVX2-NEXT: vpslld $31, %xmm4, %xmm41360; AVX2-NEXT: vpmovsxdq %xmm4, %ymm41361; AVX2-NEXT: vpmovzxwd {{.*#+}} xmm9 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero1362; AVX2-NEXT: vpslld $31, %xmm9, %xmm91363; AVX2-NEXT: vpmovsxdq %xmm9, %ymm91364; AVX2-NEXT: vblendvpd %ymm9, %ymm2, %ymm6, %ymm21365; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]1366; AVX2-NEXT: vpslld $31, %xmm5, %xmm51367; AVX2-NEXT: vpmovsxdq %xmm5, %ymm51368; AVX2-NEXT: vblendvpd %ymm5, %ymm3, %ymm7, %ymm31369; AVX2-NEXT: vpmaskmovq %ymm0, %ymm4, (%rdi)1370; AVX2-NEXT: vpmaskmovq %ymm1, %ymm8, 32(%rdi)1371; AVX2-NEXT: vmovapd %ymm3, 32(%rsi)1372; AVX2-NEXT: vmovapd %ymm2, (%rsi)1373; AVX2-NEXT: vzeroupper1374; AVX2-NEXT: retq1375;1376; AVX512-LABEL: test_masked_store_multiple_v8i64:1377; AVX512: # %bb.0:1378; AVX512-NEXT: vpsllw $15, %xmm2, %xmm21379; AVX512-NEXT: vpmovw2m %xmm2, %k11380; AVX512-NEXT: vpsllw $15, %xmm3, %xmm21381; AVX512-NEXT: vmovdqu64 (%rsi), %zmm31382; AVX512-NEXT: vpmovw2m %xmm2, %k21383; AVX512-NEXT: vmovdqa64 %zmm1, %zmm3 {%k2}1384; AVX512-NEXT: vmovdqu64 %zmm0, (%rdi) {%k1}1385; AVX512-NEXT: vmovdqu64 %zmm3, (%rsi)1386; AVX512-NEXT: vzeroupper1387; AVX512-NEXT: retq1388 %load = load <8 x i64>, ptr %ptr1, align 321389 %load2 = load <8 x i64>, ptr %ptr2, align 321390 %sel = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> %load1391 %sel2 = select <8 x i1> %mask2, <8 x i64> %y, <8 x i64> %load21392 store <8 x i64> %sel, ptr %ptr1, align 321393 store <8 x i64> %sel2, ptr %ptr2, align 321394 ret void1395}1396 1397define void @test_masked_store_unaligned_v4i32(<4 x i32> %data, ptr %ptr, <4 x i1> %mask) {1398; AVX-LABEL: test_masked_store_unaligned_v4i32:1399; AVX: # %bb.0:1400; AVX-NEXT: vpslld $31, %xmm1, %xmm11401; AVX-NEXT: vmaskmovps %xmm0, %xmm1, 1(%rdi)1402; AVX-NEXT: retq1403;1404; AVX2-LABEL: test_masked_store_unaligned_v4i32:1405; AVX2: # %bb.0:1406; AVX2-NEXT: vpslld $31, %xmm1, %xmm11407; AVX2-NEXT: vpmaskmovd %xmm0, %xmm1, 1(%rdi)1408; AVX2-NEXT: retq1409;1410; AVX512-LABEL: test_masked_store_unaligned_v4i32:1411; AVX512: # %bb.0:1412; AVX512-NEXT: vpslld $31, %xmm1, %xmm11413; AVX512-NEXT: vpmovd2m %xmm1, %k11414; AVX512-NEXT: vmovdqu32 %xmm0, 1(%rdi) {%k1}1415; AVX512-NEXT: retq1416 %ptr_i8 = getelementptr i8, ptr %ptr, i32 11417 %ptr_vec = bitcast ptr %ptr_i8 to ptr1418 %load = load <4 x i32>, ptr %ptr_vec, align 11419 %sel = select <4 x i1> %mask, <4 x i32> %data, <4 x i32> %load1420 store <4 x i32> %sel, ptr %ptr_vec, align 11421 ret void1422}1423 1424define void @test_masked_store_unaligned_v4i64(<4 x i64> %data, ptr %ptr, <4 x i1> %mask) {1425; AVX-LABEL: test_masked_store_unaligned_v4i64:1426; AVX: # %bb.0:1427; AVX-NEXT: vpslld $31, %xmm1, %xmm11428; AVX-NEXT: vpmovsxdq %xmm1, %xmm21429; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]1430; AVX-NEXT: vpmovsxdq %xmm1, %xmm11431; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm11432; AVX-NEXT: vmaskmovpd %ymm0, %ymm1, 1(%rdi)1433; AVX-NEXT: vzeroupper1434; AVX-NEXT: retq1435;1436; AVX2-LABEL: test_masked_store_unaligned_v4i64:1437; AVX2: # %bb.0:1438; AVX2-NEXT: vpslld $31, %xmm1, %xmm11439; AVX2-NEXT: vpmovsxdq %xmm1, %ymm11440; AVX2-NEXT: vpmaskmovq %ymm0, %ymm1, 1(%rdi)1441; AVX2-NEXT: vzeroupper1442; AVX2-NEXT: retq1443;1444; AVX512-LABEL: test_masked_store_unaligned_v4i64:1445; AVX512: # %bb.0:1446; AVX512-NEXT: vpslld $31, %xmm1, %xmm11447; AVX512-NEXT: vpmovd2m %xmm1, %k11448; AVX512-NEXT: vmovdqu64 %ymm0, 1(%rdi) {%k1}1449; AVX512-NEXT: vzeroupper1450; AVX512-NEXT: retq1451 %ptr_i8 = getelementptr i8, ptr %ptr, i64 11452 %ptr_vec = bitcast ptr %ptr_i8 to ptr1453 %load = load <4 x i64>, ptr %ptr_vec, align 11454 %sel = select <4 x i1> %mask, <4 x i64> %data, <4 x i64> %load1455 store <4 x i64> %sel, ptr %ptr_vec, align 11456 ret void1457}1458 1459define void @test_masked_store_unaligned_v8i32(<8 x i32> %data, ptr %ptr, <8 x i1> %mask) {1460; AVX-LABEL: test_masked_store_unaligned_v8i32:1461; AVX: # %bb.0:1462; AVX-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero1463; AVX-NEXT: vpslld $31, %xmm2, %xmm21464; AVX-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]1465; AVX-NEXT: vpslld $31, %xmm1, %xmm11466; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm11467; AVX-NEXT: vmaskmovps %ymm0, %ymm1, 1(%rdi)1468; AVX-NEXT: vzeroupper1469; AVX-NEXT: retq1470;1471; AVX2-LABEL: test_masked_store_unaligned_v8i32:1472; AVX2: # %bb.0:1473; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1474; AVX2-NEXT: vpslld $31, %ymm1, %ymm11475; AVX2-NEXT: vpmaskmovd %ymm0, %ymm1, 1(%rdi)1476; AVX2-NEXT: vzeroupper1477; AVX2-NEXT: retq1478;1479; AVX512-LABEL: test_masked_store_unaligned_v8i32:1480; AVX512: # %bb.0:1481; AVX512-NEXT: vpsllw $15, %xmm1, %xmm11482; AVX512-NEXT: vpmovw2m %xmm1, %k11483; AVX512-NEXT: vmovdqu32 %ymm0, 1(%rdi) {%k1}1484; AVX512-NEXT: vzeroupper1485; AVX512-NEXT: retq1486 %ptr_i8 = getelementptr i8, ptr %ptr, i32 11487 %ptr_vec = bitcast ptr %ptr_i8 to ptr1488 %load = load <8 x i32>, ptr %ptr_vec, align 11489 %sel = select <8 x i1> %mask, <8 x i32> %data, <8 x i32> %load1490 store <8 x i32> %sel, ptr %ptr_vec, align 11491 ret void1492}1493 1494define void @test_masked_store_unaligned_v8i64(<8 x i64> %data, ptr %ptr, <8 x i1> %mask) {1495; AVX-LABEL: test_masked_store_unaligned_v8i64:1496; AVX: # %bb.0:1497; AVX-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7]1498; AVX-NEXT: vpslld $31, %xmm3, %xmm31499; AVX-NEXT: vpmovsxdq %xmm3, %xmm41500; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]1501; AVX-NEXT: vpmovsxdq %xmm3, %xmm31502; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm31503; AVX-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero1504; AVX-NEXT: vpslld $31, %xmm2, %xmm21505; AVX-NEXT: vpmovsxdq %xmm2, %xmm41506; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]1507; AVX-NEXT: vpmovsxdq %xmm2, %xmm21508; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm4, %ymm21509; AVX-NEXT: vmaskmovpd %ymm0, %ymm2, 1(%rdi)1510; AVX-NEXT: vmaskmovpd %ymm1, %ymm3, 33(%rdi)1511; AVX-NEXT: vzeroupper1512; AVX-NEXT: retq1513;1514; AVX2-LABEL: test_masked_store_unaligned_v8i64:1515; AVX2: # %bb.0:1516; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7]1517; AVX2-NEXT: vpslld $31, %xmm3, %xmm31518; AVX2-NEXT: vpmovsxdq %xmm3, %ymm31519; AVX2-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero1520; AVX2-NEXT: vpslld $31, %xmm2, %xmm21521; AVX2-NEXT: vpmovsxdq %xmm2, %ymm21522; AVX2-NEXT: vpmaskmovq %ymm0, %ymm2, 1(%rdi)1523; AVX2-NEXT: vpmaskmovq %ymm1, %ymm3, 33(%rdi)1524; AVX2-NEXT: vzeroupper1525; AVX2-NEXT: retq1526;1527; AVX512-LABEL: test_masked_store_unaligned_v8i64:1528; AVX512: # %bb.0:1529; AVX512-NEXT: vpsllw $15, %xmm1, %xmm11530; AVX512-NEXT: vpmovw2m %xmm1, %k11531; AVX512-NEXT: vmovdqu64 %zmm0, 1(%rdi) {%k1}1532; AVX512-NEXT: vzeroupper1533; AVX512-NEXT: retq1534 %ptr_i8 = getelementptr i8, ptr %ptr, i64 11535 %ptr_vec = bitcast ptr %ptr_i8 to ptr1536 %load = load <8 x i64>, ptr %ptr_vec, align 11537 %sel = select <8 x i1> %mask, <8 x i64> %data, <8 x i64> %load1538 store <8 x i64> %sel, ptr %ptr_vec, align 11539 ret void1540}1541