6272 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=sse2 | FileCheck %s --check-prefixes=SSE,SSE23; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=sse4.2 | FileCheck %s --check-prefixes=SSE,SSE44; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX15; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx2 | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX26; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx512f | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F7; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx512f,avx512dq,avx512vl | FileCheck %s --check-prefixes=AVX,AVX512,AVX512VL,AVX512VLDQ8; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx512f,avx512bw,avx512vl | FileCheck %s --check-prefixes=AVX,AVX512,AVX512VL,AVX512VLBW9; RUN: llc < %s -mtriple=i686-apple-darwin -mattr=avx512f,avx512bw,avx512dq,avx512vl | FileCheck %s --check-prefixes=X86-AVX51210 11;12; vXf6413;14 15define void @store_v1f64_v1i64(<1 x i64> %trigger, ptr %addr, <1 x double> %val) nounwind {16; SSE-LABEL: store_v1f64_v1i64:17; SSE: ## %bb.0:18; SSE-NEXT: testq %rdi, %rdi19; SSE-NEXT: jns LBB0_220; SSE-NEXT: ## %bb.1: ## %cond.store21; SSE-NEXT: movsd %xmm0, (%rsi)22; SSE-NEXT: LBB0_2: ## %else23; SSE-NEXT: retq24;25; AVX-LABEL: store_v1f64_v1i64:26; AVX: ## %bb.0:27; AVX-NEXT: testq %rdi, %rdi28; AVX-NEXT: jns LBB0_229; AVX-NEXT: ## %bb.1: ## %cond.store30; AVX-NEXT: vmovsd %xmm0, (%rsi)31; AVX-NEXT: LBB0_2: ## %else32; AVX-NEXT: retq33;34; X86-AVX512-LABEL: store_v1f64_v1i64:35; X86-AVX512: ## %bb.0:36; X86-AVX512-NEXT: cmpl $0, {{[0-9]+}}(%esp)37; X86-AVX512-NEXT: jns LBB0_238; X86-AVX512-NEXT: ## %bb.1: ## %cond.store39; X86-AVX512-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero40; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax41; X86-AVX512-NEXT: vmovsd %xmm0, (%eax)42; X86-AVX512-NEXT: LBB0_2: ## %else43; X86-AVX512-NEXT: retl44 %mask = icmp slt <1 x i64> %trigger, zeroinitializer45 call void @llvm.masked.store.v1f64.p0(<1 x double> %val, ptr %addr, i32 4, <1 x i1> %mask)46 ret void47}48 49define void @store_v2f64_v2i64(<2 x i64> %trigger, ptr %addr, <2 x double> %val) nounwind {50; SSE-LABEL: store_v2f64_v2i64:51; SSE: ## %bb.0:52; SSE-NEXT: movmskpd %xmm0, %eax53; SSE-NEXT: testb $1, %al54; SSE-NEXT: jne LBB1_155; SSE-NEXT: ## %bb.2: ## %else56; SSE-NEXT: testb $2, %al57; SSE-NEXT: jne LBB1_358; SSE-NEXT: LBB1_4: ## %else259; SSE-NEXT: retq60; SSE-NEXT: LBB1_1: ## %cond.store61; SSE-NEXT: movlps %xmm1, (%rdi)62; SSE-NEXT: testb $2, %al63; SSE-NEXT: je LBB1_464; SSE-NEXT: LBB1_3: ## %cond.store165; SSE-NEXT: movhps %xmm1, 8(%rdi)66; SSE-NEXT: retq67;68; AVX1OR2-LABEL: store_v2f64_v2i64:69; AVX1OR2: ## %bb.0:70; AVX1OR2-NEXT: vmaskmovpd %xmm1, %xmm0, (%rdi)71; AVX1OR2-NEXT: retq72;73; AVX512F-LABEL: store_v2f64_v2i64:74; AVX512F: ## %bb.0:75; AVX512F-NEXT: ## kill: def $xmm1 killed $xmm1 def $zmm176; AVX512F-NEXT: ## kill: def $xmm0 killed $xmm0 def $zmm077; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm278; AVX512F-NEXT: vpcmpgtq %zmm0, %zmm2, %k079; AVX512F-NEXT: kshiftlw $14, %k0, %k080; AVX512F-NEXT: kshiftrw $14, %k0, %k181; AVX512F-NEXT: vmovupd %zmm1, (%rdi) {%k1}82; AVX512F-NEXT: vzeroupper83; AVX512F-NEXT: retq84;85; AVX512VLDQ-LABEL: store_v2f64_v2i64:86; AVX512VLDQ: ## %bb.0:87; AVX512VLDQ-NEXT: vpmovq2m %xmm0, %k188; AVX512VLDQ-NEXT: vmovupd %xmm1, (%rdi) {%k1}89; AVX512VLDQ-NEXT: retq90;91; AVX512VLBW-LABEL: store_v2f64_v2i64:92; AVX512VLBW: ## %bb.0:93; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm294; AVX512VLBW-NEXT: vpcmpgtq %xmm0, %xmm2, %k195; AVX512VLBW-NEXT: vmovupd %xmm1, (%rdi) {%k1}96; AVX512VLBW-NEXT: retq97;98; X86-AVX512-LABEL: store_v2f64_v2i64:99; X86-AVX512: ## %bb.0:100; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax101; X86-AVX512-NEXT: vpmovq2m %xmm0, %k1102; X86-AVX512-NEXT: vmovupd %xmm1, (%eax) {%k1}103; X86-AVX512-NEXT: retl104 %mask = icmp slt <2 x i64> %trigger, zeroinitializer105 call void @llvm.masked.store.v2f64.p0(<2 x double> %val, ptr %addr, i32 4, <2 x i1> %mask)106 ret void107}108 109define void @store_v4f64_v4i64(<4 x i64> %trigger, ptr %addr, <4 x double> %val) nounwind {110; SSE2-LABEL: store_v4f64_v4i64:111; SSE2: ## %bb.0:112; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]113; SSE2-NEXT: movmskps %xmm0, %eax114; SSE2-NEXT: testb $1, %al115; SSE2-NEXT: jne LBB2_1116; SSE2-NEXT: ## %bb.2: ## %else117; SSE2-NEXT: testb $2, %al118; SSE2-NEXT: jne LBB2_3119; SSE2-NEXT: LBB2_4: ## %else2120; SSE2-NEXT: testb $4, %al121; SSE2-NEXT: jne LBB2_5122; SSE2-NEXT: LBB2_6: ## %else4123; SSE2-NEXT: testb $8, %al124; SSE2-NEXT: jne LBB2_7125; SSE2-NEXT: LBB2_8: ## %else6126; SSE2-NEXT: retq127; SSE2-NEXT: LBB2_1: ## %cond.store128; SSE2-NEXT: movlps %xmm2, (%rdi)129; SSE2-NEXT: testb $2, %al130; SSE2-NEXT: je LBB2_4131; SSE2-NEXT: LBB2_3: ## %cond.store1132; SSE2-NEXT: movhps %xmm2, 8(%rdi)133; SSE2-NEXT: testb $4, %al134; SSE2-NEXT: je LBB2_6135; SSE2-NEXT: LBB2_5: ## %cond.store3136; SSE2-NEXT: movlps %xmm3, 16(%rdi)137; SSE2-NEXT: testb $8, %al138; SSE2-NEXT: je LBB2_8139; SSE2-NEXT: LBB2_7: ## %cond.store5140; SSE2-NEXT: movhps %xmm3, 24(%rdi)141; SSE2-NEXT: retq142;143; SSE4-LABEL: store_v4f64_v4i64:144; SSE4: ## %bb.0:145; SSE4-NEXT: packssdw %xmm1, %xmm0146; SSE4-NEXT: movmskps %xmm0, %eax147; SSE4-NEXT: testb $1, %al148; SSE4-NEXT: jne LBB2_1149; SSE4-NEXT: ## %bb.2: ## %else150; SSE4-NEXT: testb $2, %al151; SSE4-NEXT: jne LBB2_3152; SSE4-NEXT: LBB2_4: ## %else2153; SSE4-NEXT: testb $4, %al154; SSE4-NEXT: jne LBB2_5155; SSE4-NEXT: LBB2_6: ## %else4156; SSE4-NEXT: testb $8, %al157; SSE4-NEXT: jne LBB2_7158; SSE4-NEXT: LBB2_8: ## %else6159; SSE4-NEXT: retq160; SSE4-NEXT: LBB2_1: ## %cond.store161; SSE4-NEXT: movlps %xmm2, (%rdi)162; SSE4-NEXT: testb $2, %al163; SSE4-NEXT: je LBB2_4164; SSE4-NEXT: LBB2_3: ## %cond.store1165; SSE4-NEXT: movhps %xmm2, 8(%rdi)166; SSE4-NEXT: testb $4, %al167; SSE4-NEXT: je LBB2_6168; SSE4-NEXT: LBB2_5: ## %cond.store3169; SSE4-NEXT: movlps %xmm3, 16(%rdi)170; SSE4-NEXT: testb $8, %al171; SSE4-NEXT: je LBB2_8172; SSE4-NEXT: LBB2_7: ## %cond.store5173; SSE4-NEXT: movhps %xmm3, 24(%rdi)174; SSE4-NEXT: retq175;176; AVX1OR2-LABEL: store_v4f64_v4i64:177; AVX1OR2: ## %bb.0:178; AVX1OR2-NEXT: vmaskmovpd %ymm1, %ymm0, (%rdi)179; AVX1OR2-NEXT: vzeroupper180; AVX1OR2-NEXT: retq181;182; AVX512F-LABEL: store_v4f64_v4i64:183; AVX512F: ## %bb.0:184; AVX512F-NEXT: ## kill: def $ymm1 killed $ymm1 def $zmm1185; AVX512F-NEXT: ## kill: def $ymm0 killed $ymm0 def $zmm0186; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2187; AVX512F-NEXT: vpcmpgtq %zmm0, %zmm2, %k0188; AVX512F-NEXT: kshiftlw $12, %k0, %k0189; AVX512F-NEXT: kshiftrw $12, %k0, %k1190; AVX512F-NEXT: vmovupd %zmm1, (%rdi) {%k1}191; AVX512F-NEXT: vzeroupper192; AVX512F-NEXT: retq193;194; AVX512VLDQ-LABEL: store_v4f64_v4i64:195; AVX512VLDQ: ## %bb.0:196; AVX512VLDQ-NEXT: vpmovq2m %ymm0, %k1197; AVX512VLDQ-NEXT: vmovupd %ymm1, (%rdi) {%k1}198; AVX512VLDQ-NEXT: vzeroupper199; AVX512VLDQ-NEXT: retq200;201; AVX512VLBW-LABEL: store_v4f64_v4i64:202; AVX512VLBW: ## %bb.0:203; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2204; AVX512VLBW-NEXT: vpcmpgtq %ymm0, %ymm2, %k1205; AVX512VLBW-NEXT: vmovupd %ymm1, (%rdi) {%k1}206; AVX512VLBW-NEXT: vzeroupper207; AVX512VLBW-NEXT: retq208;209; X86-AVX512-LABEL: store_v4f64_v4i64:210; X86-AVX512: ## %bb.0:211; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax212; X86-AVX512-NEXT: vpmovq2m %ymm0, %k1213; X86-AVX512-NEXT: vmovupd %ymm1, (%eax) {%k1}214; X86-AVX512-NEXT: vzeroupper215; X86-AVX512-NEXT: retl216 %mask = icmp slt <4 x i64> %trigger, zeroinitializer217 call void @llvm.masked.store.v4f64.p0(<4 x double> %val, ptr %addr, i32 4, <4 x i1> %mask)218 ret void219}220 221;222; vXf32223;224 225define void @store_v2f32_v2i32(<2 x i32> %trigger, ptr %addr, <2 x float> %val) nounwind {226; SSE2-LABEL: store_v2f32_v2i32:227; SSE2: ## %bb.0:228; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]229; SSE2-NEXT: pxor %xmm2, %xmm2230; SSE2-NEXT: pcmpeqd %xmm0, %xmm2231; SSE2-NEXT: movmskpd %xmm2, %eax232; SSE2-NEXT: testb $1, %al233; SSE2-NEXT: jne LBB3_1234; SSE2-NEXT: ## %bb.2: ## %else235; SSE2-NEXT: testb $2, %al236; SSE2-NEXT: jne LBB3_3237; SSE2-NEXT: LBB3_4: ## %else2238; SSE2-NEXT: retq239; SSE2-NEXT: LBB3_1: ## %cond.store240; SSE2-NEXT: movss %xmm1, (%rdi)241; SSE2-NEXT: testb $2, %al242; SSE2-NEXT: je LBB3_4243; SSE2-NEXT: LBB3_3: ## %cond.store1244; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]245; SSE2-NEXT: movss %xmm1, 4(%rdi)246; SSE2-NEXT: retq247;248; SSE4-LABEL: store_v2f32_v2i32:249; SSE4: ## %bb.0:250; SSE4-NEXT: pxor %xmm2, %xmm2251; SSE4-NEXT: pcmpeqd %xmm0, %xmm2252; SSE4-NEXT: pmovsxdq %xmm2, %xmm0253; SSE4-NEXT: movmskpd %xmm0, %eax254; SSE4-NEXT: testb $1, %al255; SSE4-NEXT: jne LBB3_1256; SSE4-NEXT: ## %bb.2: ## %else257; SSE4-NEXT: testb $2, %al258; SSE4-NEXT: jne LBB3_3259; SSE4-NEXT: LBB3_4: ## %else2260; SSE4-NEXT: retq261; SSE4-NEXT: LBB3_1: ## %cond.store262; SSE4-NEXT: movss %xmm1, (%rdi)263; SSE4-NEXT: testb $2, %al264; SSE4-NEXT: je LBB3_4265; SSE4-NEXT: LBB3_3: ## %cond.store1266; SSE4-NEXT: extractps $1, %xmm1, 4(%rdi)267; SSE4-NEXT: retq268;269; AVX1OR2-LABEL: store_v2f32_v2i32:270; AVX1OR2: ## %bb.0:271; AVX1OR2-NEXT: vpxor %xmm2, %xmm2, %xmm2272; AVX1OR2-NEXT: vpcmpeqd %xmm2, %xmm0, %xmm0273; AVX1OR2-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero274; AVX1OR2-NEXT: vmaskmovps %xmm1, %xmm0, (%rdi)275; AVX1OR2-NEXT: retq276;277; AVX512F-LABEL: store_v2f32_v2i32:278; AVX512F: ## %bb.0:279; AVX512F-NEXT: ## kill: def $xmm1 killed $xmm1 def $zmm1280; AVX512F-NEXT: ## kill: def $xmm0 killed $xmm0 def $zmm0281; AVX512F-NEXT: vptestnmd %zmm0, %zmm0, %k0282; AVX512F-NEXT: kshiftlw $14, %k0, %k0283; AVX512F-NEXT: kshiftrw $14, %k0, %k1284; AVX512F-NEXT: vmovups %zmm1, (%rdi) {%k1}285; AVX512F-NEXT: vzeroupper286; AVX512F-NEXT: retq287;288; AVX512VLDQ-LABEL: store_v2f32_v2i32:289; AVX512VLDQ: ## %bb.0:290; AVX512VLDQ-NEXT: vptestnmd %xmm0, %xmm0, %k0291; AVX512VLDQ-NEXT: kshiftlb $6, %k0, %k0292; AVX512VLDQ-NEXT: kshiftrb $6, %k0, %k1293; AVX512VLDQ-NEXT: vmovups %xmm1, (%rdi) {%k1}294; AVX512VLDQ-NEXT: retq295;296; AVX512VLBW-LABEL: store_v2f32_v2i32:297; AVX512VLBW: ## %bb.0:298; AVX512VLBW-NEXT: vptestnmd %xmm0, %xmm0, %k0299; AVX512VLBW-NEXT: kshiftlw $14, %k0, %k0300; AVX512VLBW-NEXT: kshiftrw $14, %k0, %k1301; AVX512VLBW-NEXT: vmovups %xmm1, (%rdi) {%k1}302; AVX512VLBW-NEXT: retq303;304; X86-AVX512-LABEL: store_v2f32_v2i32:305; X86-AVX512: ## %bb.0:306; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax307; X86-AVX512-NEXT: vptestnmd %xmm0, %xmm0, %k0308; X86-AVX512-NEXT: kshiftlb $6, %k0, %k0309; X86-AVX512-NEXT: kshiftrb $6, %k0, %k1310; X86-AVX512-NEXT: vmovups %xmm1, (%eax) {%k1}311; X86-AVX512-NEXT: retl312 %mask = icmp eq <2 x i32> %trigger, zeroinitializer313 call void @llvm.masked.store.v2f32.p0(<2 x float> %val, ptr %addr, i32 4, <2 x i1> %mask)314 ret void315}316 317define void @store_v4f32_v4i32(<4 x float> %x, ptr %ptr, <4 x float> %y, <4 x i32> %mask) nounwind {318; SSE2-LABEL: store_v4f32_v4i32:319; SSE2: ## %bb.0:320; SSE2-NEXT: movmskps %xmm2, %eax321; SSE2-NEXT: testb $1, %al322; SSE2-NEXT: jne LBB4_1323; SSE2-NEXT: ## %bb.2: ## %else324; SSE2-NEXT: testb $2, %al325; SSE2-NEXT: jne LBB4_3326; SSE2-NEXT: LBB4_4: ## %else2327; SSE2-NEXT: testb $4, %al328; SSE2-NEXT: jne LBB4_5329; SSE2-NEXT: LBB4_6: ## %else4330; SSE2-NEXT: testb $8, %al331; SSE2-NEXT: jne LBB4_7332; SSE2-NEXT: LBB4_8: ## %else6333; SSE2-NEXT: retq334; SSE2-NEXT: LBB4_1: ## %cond.store335; SSE2-NEXT: movss %xmm0, (%rdi)336; SSE2-NEXT: testb $2, %al337; SSE2-NEXT: je LBB4_4338; SSE2-NEXT: LBB4_3: ## %cond.store1339; SSE2-NEXT: movaps %xmm0, %xmm1340; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]341; SSE2-NEXT: movss %xmm1, 4(%rdi)342; SSE2-NEXT: testb $4, %al343; SSE2-NEXT: je LBB4_6344; SSE2-NEXT: LBB4_5: ## %cond.store3345; SSE2-NEXT: movaps %xmm0, %xmm1346; SSE2-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]347; SSE2-NEXT: movss %xmm1, 8(%rdi)348; SSE2-NEXT: testb $8, %al349; SSE2-NEXT: je LBB4_8350; SSE2-NEXT: LBB4_7: ## %cond.store5351; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]352; SSE2-NEXT: movss %xmm0, 12(%rdi)353; SSE2-NEXT: retq354;355; SSE4-LABEL: store_v4f32_v4i32:356; SSE4: ## %bb.0:357; SSE4-NEXT: movmskps %xmm2, %eax358; SSE4-NEXT: testb $1, %al359; SSE4-NEXT: jne LBB4_1360; SSE4-NEXT: ## %bb.2: ## %else361; SSE4-NEXT: testb $2, %al362; SSE4-NEXT: jne LBB4_3363; SSE4-NEXT: LBB4_4: ## %else2364; SSE4-NEXT: testb $4, %al365; SSE4-NEXT: jne LBB4_5366; SSE4-NEXT: LBB4_6: ## %else4367; SSE4-NEXT: testb $8, %al368; SSE4-NEXT: jne LBB4_7369; SSE4-NEXT: LBB4_8: ## %else6370; SSE4-NEXT: retq371; SSE4-NEXT: LBB4_1: ## %cond.store372; SSE4-NEXT: movss %xmm0, (%rdi)373; SSE4-NEXT: testb $2, %al374; SSE4-NEXT: je LBB4_4375; SSE4-NEXT: LBB4_3: ## %cond.store1376; SSE4-NEXT: extractps $1, %xmm0, 4(%rdi)377; SSE4-NEXT: testb $4, %al378; SSE4-NEXT: je LBB4_6379; SSE4-NEXT: LBB4_5: ## %cond.store3380; SSE4-NEXT: extractps $2, %xmm0, 8(%rdi)381; SSE4-NEXT: testb $8, %al382; SSE4-NEXT: je LBB4_8383; SSE4-NEXT: LBB4_7: ## %cond.store5384; SSE4-NEXT: extractps $3, %xmm0, 12(%rdi)385; SSE4-NEXT: retq386;387; AVX1OR2-LABEL: store_v4f32_v4i32:388; AVX1OR2: ## %bb.0:389; AVX1OR2-NEXT: vmaskmovps %xmm0, %xmm2, (%rdi)390; AVX1OR2-NEXT: retq391;392; AVX512F-LABEL: store_v4f32_v4i32:393; AVX512F: ## %bb.0:394; AVX512F-NEXT: ## kill: def $xmm2 killed $xmm2 def $zmm2395; AVX512F-NEXT: ## kill: def $xmm0 killed $xmm0 def $zmm0396; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1397; AVX512F-NEXT: vpcmpgtd %zmm2, %zmm1, %k0398; AVX512F-NEXT: kshiftlw $12, %k0, %k0399; AVX512F-NEXT: kshiftrw $12, %k0, %k1400; AVX512F-NEXT: vmovups %zmm0, (%rdi) {%k1}401; AVX512F-NEXT: vzeroupper402; AVX512F-NEXT: retq403;404; AVX512VLDQ-LABEL: store_v4f32_v4i32:405; AVX512VLDQ: ## %bb.0:406; AVX512VLDQ-NEXT: vpmovd2m %xmm2, %k1407; AVX512VLDQ-NEXT: vmovups %xmm0, (%rdi) {%k1}408; AVX512VLDQ-NEXT: retq409;410; AVX512VLBW-LABEL: store_v4f32_v4i32:411; AVX512VLBW: ## %bb.0:412; AVX512VLBW-NEXT: vpxor %xmm1, %xmm1, %xmm1413; AVX512VLBW-NEXT: vpcmpgtd %xmm2, %xmm1, %k1414; AVX512VLBW-NEXT: vmovups %xmm0, (%rdi) {%k1}415; AVX512VLBW-NEXT: retq416;417; X86-AVX512-LABEL: store_v4f32_v4i32:418; X86-AVX512: ## %bb.0:419; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax420; X86-AVX512-NEXT: vpmovd2m %xmm2, %k1421; X86-AVX512-NEXT: vmovups %xmm0, (%eax) {%k1}422; X86-AVX512-NEXT: retl423 %bool_mask = icmp slt <4 x i32> %mask, zeroinitializer424 call void @llvm.masked.store.v4f32.p0(<4 x float> %x, ptr %ptr, i32 1, <4 x i1> %bool_mask)425 ret void426}427 428define void @store_v8f32_v8i32(<8 x float> %x, ptr %ptr, <8 x float> %y, <8 x i32> %mask) nounwind {429; SSE2-LABEL: store_v8f32_v8i32:430; SSE2: ## %bb.0:431; SSE2-NEXT: packssdw %xmm5, %xmm4432; SSE2-NEXT: packsswb %xmm4, %xmm4433; SSE2-NEXT: pmovmskb %xmm4, %eax434; SSE2-NEXT: testb $1, %al435; SSE2-NEXT: jne LBB5_1436; SSE2-NEXT: ## %bb.2: ## %else437; SSE2-NEXT: testb $2, %al438; SSE2-NEXT: jne LBB5_3439; SSE2-NEXT: LBB5_4: ## %else2440; SSE2-NEXT: testb $4, %al441; SSE2-NEXT: jne LBB5_5442; SSE2-NEXT: LBB5_6: ## %else4443; SSE2-NEXT: testb $8, %al444; SSE2-NEXT: jne LBB5_7445; SSE2-NEXT: LBB5_8: ## %else6446; SSE2-NEXT: testb $16, %al447; SSE2-NEXT: jne LBB5_9448; SSE2-NEXT: LBB5_10: ## %else8449; SSE2-NEXT: testb $32, %al450; SSE2-NEXT: jne LBB5_11451; SSE2-NEXT: LBB5_12: ## %else10452; SSE2-NEXT: testb $64, %al453; SSE2-NEXT: jne LBB5_13454; SSE2-NEXT: LBB5_14: ## %else12455; SSE2-NEXT: testb $-128, %al456; SSE2-NEXT: jne LBB5_15457; SSE2-NEXT: LBB5_16: ## %else14458; SSE2-NEXT: retq459; SSE2-NEXT: LBB5_1: ## %cond.store460; SSE2-NEXT: movd %xmm0, (%rdi)461; SSE2-NEXT: testb $2, %al462; SSE2-NEXT: je LBB5_4463; SSE2-NEXT: LBB5_3: ## %cond.store1464; SSE2-NEXT: movdqa %xmm0, %xmm2465; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[1,1]466; SSE2-NEXT: movss %xmm2, 4(%rdi)467; SSE2-NEXT: testb $4, %al468; SSE2-NEXT: je LBB5_6469; SSE2-NEXT: LBB5_5: ## %cond.store3470; SSE2-NEXT: movaps %xmm0, %xmm2471; SSE2-NEXT: unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]472; SSE2-NEXT: movss %xmm2, 8(%rdi)473; SSE2-NEXT: testb $8, %al474; SSE2-NEXT: je LBB5_8475; SSE2-NEXT: LBB5_7: ## %cond.store5476; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]477; SSE2-NEXT: movss %xmm0, 12(%rdi)478; SSE2-NEXT: testb $16, %al479; SSE2-NEXT: je LBB5_10480; SSE2-NEXT: LBB5_9: ## %cond.store7481; SSE2-NEXT: movss %xmm1, 16(%rdi)482; SSE2-NEXT: testb $32, %al483; SSE2-NEXT: je LBB5_12484; SSE2-NEXT: LBB5_11: ## %cond.store9485; SSE2-NEXT: movaps %xmm1, %xmm0486; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]487; SSE2-NEXT: movss %xmm0, 20(%rdi)488; SSE2-NEXT: testb $64, %al489; SSE2-NEXT: je LBB5_14490; SSE2-NEXT: LBB5_13: ## %cond.store11491; SSE2-NEXT: movaps %xmm1, %xmm0492; SSE2-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]493; SSE2-NEXT: movss %xmm0, 24(%rdi)494; SSE2-NEXT: testb $-128, %al495; SSE2-NEXT: je LBB5_16496; SSE2-NEXT: LBB5_15: ## %cond.store13497; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]498; SSE2-NEXT: movss %xmm1, 28(%rdi)499; SSE2-NEXT: retq500;501; SSE4-LABEL: store_v8f32_v8i32:502; SSE4: ## %bb.0:503; SSE4-NEXT: packssdw %xmm5, %xmm4504; SSE4-NEXT: packsswb %xmm4, %xmm4505; SSE4-NEXT: pmovmskb %xmm4, %eax506; SSE4-NEXT: testb $1, %al507; SSE4-NEXT: jne LBB5_1508; SSE4-NEXT: ## %bb.2: ## %else509; SSE4-NEXT: testb $2, %al510; SSE4-NEXT: jne LBB5_3511; SSE4-NEXT: LBB5_4: ## %else2512; SSE4-NEXT: testb $4, %al513; SSE4-NEXT: jne LBB5_5514; SSE4-NEXT: LBB5_6: ## %else4515; SSE4-NEXT: testb $8, %al516; SSE4-NEXT: jne LBB5_7517; SSE4-NEXT: LBB5_8: ## %else6518; SSE4-NEXT: testb $16, %al519; SSE4-NEXT: jne LBB5_9520; SSE4-NEXT: LBB5_10: ## %else8521; SSE4-NEXT: testb $32, %al522; SSE4-NEXT: jne LBB5_11523; SSE4-NEXT: LBB5_12: ## %else10524; SSE4-NEXT: testb $64, %al525; SSE4-NEXT: jne LBB5_13526; SSE4-NEXT: LBB5_14: ## %else12527; SSE4-NEXT: testb $-128, %al528; SSE4-NEXT: jne LBB5_15529; SSE4-NEXT: LBB5_16: ## %else14530; SSE4-NEXT: retq531; SSE4-NEXT: LBB5_1: ## %cond.store532; SSE4-NEXT: movd %xmm0, (%rdi)533; SSE4-NEXT: testb $2, %al534; SSE4-NEXT: je LBB5_4535; SSE4-NEXT: LBB5_3: ## %cond.store1536; SSE4-NEXT: pextrd $1, %xmm0, 4(%rdi)537; SSE4-NEXT: testb $4, %al538; SSE4-NEXT: je LBB5_6539; SSE4-NEXT: LBB5_5: ## %cond.store3540; SSE4-NEXT: pextrd $2, %xmm0, 8(%rdi)541; SSE4-NEXT: testb $8, %al542; SSE4-NEXT: je LBB5_8543; SSE4-NEXT: LBB5_7: ## %cond.store5544; SSE4-NEXT: pextrd $3, %xmm0, 12(%rdi)545; SSE4-NEXT: testb $16, %al546; SSE4-NEXT: je LBB5_10547; SSE4-NEXT: LBB5_9: ## %cond.store7548; SSE4-NEXT: movss %xmm1, 16(%rdi)549; SSE4-NEXT: testb $32, %al550; SSE4-NEXT: je LBB5_12551; SSE4-NEXT: LBB5_11: ## %cond.store9552; SSE4-NEXT: extractps $1, %xmm1, 20(%rdi)553; SSE4-NEXT: testb $64, %al554; SSE4-NEXT: je LBB5_14555; SSE4-NEXT: LBB5_13: ## %cond.store11556; SSE4-NEXT: extractps $2, %xmm1, 24(%rdi)557; SSE4-NEXT: testb $-128, %al558; SSE4-NEXT: je LBB5_16559; SSE4-NEXT: LBB5_15: ## %cond.store13560; SSE4-NEXT: extractps $3, %xmm1, 28(%rdi)561; SSE4-NEXT: retq562;563; AVX1OR2-LABEL: store_v8f32_v8i32:564; AVX1OR2: ## %bb.0:565; AVX1OR2-NEXT: vmaskmovps %ymm0, %ymm2, (%rdi)566; AVX1OR2-NEXT: vzeroupper567; AVX1OR2-NEXT: retq568;569; AVX512F-LABEL: store_v8f32_v8i32:570; AVX512F: ## %bb.0:571; AVX512F-NEXT: ## kill: def $ymm2 killed $ymm2 def $zmm2572; AVX512F-NEXT: ## kill: def $ymm0 killed $ymm0 def $zmm0573; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1574; AVX512F-NEXT: vpcmpgtd %zmm2, %zmm1, %k0575; AVX512F-NEXT: kshiftlw $8, %k0, %k0576; AVX512F-NEXT: kshiftrw $8, %k0, %k1577; AVX512F-NEXT: vmovups %zmm0, (%rdi) {%k1}578; AVX512F-NEXT: vzeroupper579; AVX512F-NEXT: retq580;581; AVX512VLDQ-LABEL: store_v8f32_v8i32:582; AVX512VLDQ: ## %bb.0:583; AVX512VLDQ-NEXT: vpmovd2m %ymm2, %k1584; AVX512VLDQ-NEXT: vmovups %ymm0, (%rdi) {%k1}585; AVX512VLDQ-NEXT: vzeroupper586; AVX512VLDQ-NEXT: retq587;588; AVX512VLBW-LABEL: store_v8f32_v8i32:589; AVX512VLBW: ## %bb.0:590; AVX512VLBW-NEXT: vpxor %xmm1, %xmm1, %xmm1591; AVX512VLBW-NEXT: vpcmpgtd %ymm2, %ymm1, %k1592; AVX512VLBW-NEXT: vmovups %ymm0, (%rdi) {%k1}593; AVX512VLBW-NEXT: vzeroupper594; AVX512VLBW-NEXT: retq595;596; X86-AVX512-LABEL: store_v8f32_v8i32:597; X86-AVX512: ## %bb.0:598; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax599; X86-AVX512-NEXT: vpmovd2m %ymm2, %k1600; X86-AVX512-NEXT: vmovups %ymm0, (%eax) {%k1}601; X86-AVX512-NEXT: vzeroupper602; X86-AVX512-NEXT: retl603 %bool_mask = icmp slt <8 x i32> %mask, zeroinitializer604 call void @llvm.masked.store.v8f32.p0(<8 x float> %x, ptr %ptr, i32 1, <8 x i1> %bool_mask)605 ret void606}607 608define void @store_v16f32_v16i32(<16 x float> %x, ptr %ptr, <16 x float> %y, <16 x i32> %mask) nounwind {609; SSE2-LABEL: store_v16f32_v16i32:610; SSE2: ## %bb.0:611; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm4612; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm5613; SSE2-NEXT: packssdw {{[0-9]+}}(%rsp), %xmm5614; SSE2-NEXT: packssdw {{[0-9]+}}(%rsp), %xmm4615; SSE2-NEXT: packsswb %xmm5, %xmm4616; SSE2-NEXT: pmovmskb %xmm4, %eax617; SSE2-NEXT: testb $1, %al618; SSE2-NEXT: jne LBB6_1619; SSE2-NEXT: ## %bb.2: ## %else620; SSE2-NEXT: testb $2, %al621; SSE2-NEXT: jne LBB6_3622; SSE2-NEXT: LBB6_4: ## %else2623; SSE2-NEXT: testb $4, %al624; SSE2-NEXT: jne LBB6_5625; SSE2-NEXT: LBB6_6: ## %else4626; SSE2-NEXT: testb $8, %al627; SSE2-NEXT: jne LBB6_7628; SSE2-NEXT: LBB6_8: ## %else6629; SSE2-NEXT: testb $16, %al630; SSE2-NEXT: jne LBB6_9631; SSE2-NEXT: LBB6_10: ## %else8632; SSE2-NEXT: testb $32, %al633; SSE2-NEXT: jne LBB6_11634; SSE2-NEXT: LBB6_12: ## %else10635; SSE2-NEXT: testb $64, %al636; SSE2-NEXT: jne LBB6_13637; SSE2-NEXT: LBB6_14: ## %else12638; SSE2-NEXT: testb %al, %al639; SSE2-NEXT: js LBB6_15640; SSE2-NEXT: LBB6_16: ## %else14641; SSE2-NEXT: testl $256, %eax ## imm = 0x100642; SSE2-NEXT: jne LBB6_17643; SSE2-NEXT: LBB6_18: ## %else16644; SSE2-NEXT: testl $512, %eax ## imm = 0x200645; SSE2-NEXT: jne LBB6_19646; SSE2-NEXT: LBB6_20: ## %else18647; SSE2-NEXT: testl $1024, %eax ## imm = 0x400648; SSE2-NEXT: jne LBB6_21649; SSE2-NEXT: LBB6_22: ## %else20650; SSE2-NEXT: testl $2048, %eax ## imm = 0x800651; SSE2-NEXT: jne LBB6_23652; SSE2-NEXT: LBB6_24: ## %else22653; SSE2-NEXT: testl $4096, %eax ## imm = 0x1000654; SSE2-NEXT: jne LBB6_25655; SSE2-NEXT: LBB6_26: ## %else24656; SSE2-NEXT: testl $8192, %eax ## imm = 0x2000657; SSE2-NEXT: jne LBB6_27658; SSE2-NEXT: LBB6_28: ## %else26659; SSE2-NEXT: testl $16384, %eax ## imm = 0x4000660; SSE2-NEXT: jne LBB6_29661; SSE2-NEXT: LBB6_30: ## %else28662; SSE2-NEXT: testl $32768, %eax ## imm = 0x8000663; SSE2-NEXT: jne LBB6_31664; SSE2-NEXT: LBB6_32: ## %else30665; SSE2-NEXT: retq666; SSE2-NEXT: LBB6_1: ## %cond.store667; SSE2-NEXT: movss %xmm0, (%rdi)668; SSE2-NEXT: testb $2, %al669; SSE2-NEXT: je LBB6_4670; SSE2-NEXT: LBB6_3: ## %cond.store1671; SSE2-NEXT: movaps %xmm0, %xmm4672; SSE2-NEXT: shufps {{.*#+}} xmm4 = xmm4[1,1],xmm0[1,1]673; SSE2-NEXT: movss %xmm4, 4(%rdi)674; SSE2-NEXT: testb $4, %al675; SSE2-NEXT: je LBB6_6676; SSE2-NEXT: LBB6_5: ## %cond.store3677; SSE2-NEXT: movaps %xmm0, %xmm4678; SSE2-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]679; SSE2-NEXT: movss %xmm4, 8(%rdi)680; SSE2-NEXT: testb $8, %al681; SSE2-NEXT: je LBB6_8682; SSE2-NEXT: LBB6_7: ## %cond.store5683; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]684; SSE2-NEXT: movss %xmm0, 12(%rdi)685; SSE2-NEXT: testb $16, %al686; SSE2-NEXT: je LBB6_10687; SSE2-NEXT: LBB6_9: ## %cond.store7688; SSE2-NEXT: movss %xmm1, 16(%rdi)689; SSE2-NEXT: testb $32, %al690; SSE2-NEXT: je LBB6_12691; SSE2-NEXT: LBB6_11: ## %cond.store9692; SSE2-NEXT: movaps %xmm1, %xmm0693; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]694; SSE2-NEXT: movss %xmm0, 20(%rdi)695; SSE2-NEXT: testb $64, %al696; SSE2-NEXT: je LBB6_14697; SSE2-NEXT: LBB6_13: ## %cond.store11698; SSE2-NEXT: movaps %xmm1, %xmm0699; SSE2-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]700; SSE2-NEXT: movss %xmm0, 24(%rdi)701; SSE2-NEXT: testb %al, %al702; SSE2-NEXT: jns LBB6_16703; SSE2-NEXT: LBB6_15: ## %cond.store13704; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]705; SSE2-NEXT: movss %xmm1, 28(%rdi)706; SSE2-NEXT: testl $256, %eax ## imm = 0x100707; SSE2-NEXT: je LBB6_18708; SSE2-NEXT: LBB6_17: ## %cond.store15709; SSE2-NEXT: movss %xmm2, 32(%rdi)710; SSE2-NEXT: testl $512, %eax ## imm = 0x200711; SSE2-NEXT: je LBB6_20712; SSE2-NEXT: LBB6_19: ## %cond.store17713; SSE2-NEXT: movaps %xmm2, %xmm0714; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm2[1,1]715; SSE2-NEXT: movss %xmm0, 36(%rdi)716; SSE2-NEXT: testl $1024, %eax ## imm = 0x400717; SSE2-NEXT: je LBB6_22718; SSE2-NEXT: LBB6_21: ## %cond.store19719; SSE2-NEXT: movaps %xmm2, %xmm0720; SSE2-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm2[1]721; SSE2-NEXT: movss %xmm0, 40(%rdi)722; SSE2-NEXT: testl $2048, %eax ## imm = 0x800723; SSE2-NEXT: je LBB6_24724; SSE2-NEXT: LBB6_23: ## %cond.store21725; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]726; SSE2-NEXT: movss %xmm2, 44(%rdi)727; SSE2-NEXT: testl $4096, %eax ## imm = 0x1000728; SSE2-NEXT: je LBB6_26729; SSE2-NEXT: LBB6_25: ## %cond.store23730; SSE2-NEXT: movss %xmm3, 48(%rdi)731; SSE2-NEXT: testl $8192, %eax ## imm = 0x2000732; SSE2-NEXT: je LBB6_28733; SSE2-NEXT: LBB6_27: ## %cond.store25734; SSE2-NEXT: movaps %xmm3, %xmm0735; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm3[1,1]736; SSE2-NEXT: movss %xmm0, 52(%rdi)737; SSE2-NEXT: testl $16384, %eax ## imm = 0x4000738; SSE2-NEXT: je LBB6_30739; SSE2-NEXT: LBB6_29: ## %cond.store27740; SSE2-NEXT: movaps %xmm3, %xmm0741; SSE2-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm3[1]742; SSE2-NEXT: movss %xmm0, 56(%rdi)743; SSE2-NEXT: testl $32768, %eax ## imm = 0x8000744; SSE2-NEXT: je LBB6_32745; SSE2-NEXT: LBB6_31: ## %cond.store29746; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]747; SSE2-NEXT: movss %xmm3, 60(%rdi)748; SSE2-NEXT: retq749;750; SSE4-LABEL: store_v16f32_v16i32:751; SSE4: ## %bb.0:752; SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm4753; SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm5754; SSE4-NEXT: packssdw {{[0-9]+}}(%rsp), %xmm5755; SSE4-NEXT: packssdw {{[0-9]+}}(%rsp), %xmm4756; SSE4-NEXT: packsswb %xmm5, %xmm4757; SSE4-NEXT: pmovmskb %xmm4, %eax758; SSE4-NEXT: testb $1, %al759; SSE4-NEXT: jne LBB6_1760; SSE4-NEXT: ## %bb.2: ## %else761; SSE4-NEXT: testb $2, %al762; SSE4-NEXT: jne LBB6_3763; SSE4-NEXT: LBB6_4: ## %else2764; SSE4-NEXT: testb $4, %al765; SSE4-NEXT: jne LBB6_5766; SSE4-NEXT: LBB6_6: ## %else4767; SSE4-NEXT: testb $8, %al768; SSE4-NEXT: jne LBB6_7769; SSE4-NEXT: LBB6_8: ## %else6770; SSE4-NEXT: testb $16, %al771; SSE4-NEXT: jne LBB6_9772; SSE4-NEXT: LBB6_10: ## %else8773; SSE4-NEXT: testb $32, %al774; SSE4-NEXT: jne LBB6_11775; SSE4-NEXT: LBB6_12: ## %else10776; SSE4-NEXT: testb $64, %al777; SSE4-NEXT: jne LBB6_13778; SSE4-NEXT: LBB6_14: ## %else12779; SSE4-NEXT: testb %al, %al780; SSE4-NEXT: js LBB6_15781; SSE4-NEXT: LBB6_16: ## %else14782; SSE4-NEXT: testl $256, %eax ## imm = 0x100783; SSE4-NEXT: jne LBB6_17784; SSE4-NEXT: LBB6_18: ## %else16785; SSE4-NEXT: testl $512, %eax ## imm = 0x200786; SSE4-NEXT: jne LBB6_19787; SSE4-NEXT: LBB6_20: ## %else18788; SSE4-NEXT: testl $1024, %eax ## imm = 0x400789; SSE4-NEXT: jne LBB6_21790; SSE4-NEXT: LBB6_22: ## %else20791; SSE4-NEXT: testl $2048, %eax ## imm = 0x800792; SSE4-NEXT: jne LBB6_23793; SSE4-NEXT: LBB6_24: ## %else22794; SSE4-NEXT: testl $4096, %eax ## imm = 0x1000795; SSE4-NEXT: jne LBB6_25796; SSE4-NEXT: LBB6_26: ## %else24797; SSE4-NEXT: testl $8192, %eax ## imm = 0x2000798; SSE4-NEXT: jne LBB6_27799; SSE4-NEXT: LBB6_28: ## %else26800; SSE4-NEXT: testl $16384, %eax ## imm = 0x4000801; SSE4-NEXT: jne LBB6_29802; SSE4-NEXT: LBB6_30: ## %else28803; SSE4-NEXT: testl $32768, %eax ## imm = 0x8000804; SSE4-NEXT: jne LBB6_31805; SSE4-NEXT: LBB6_32: ## %else30806; SSE4-NEXT: retq807; SSE4-NEXT: LBB6_1: ## %cond.store808; SSE4-NEXT: movss %xmm0, (%rdi)809; SSE4-NEXT: testb $2, %al810; SSE4-NEXT: je LBB6_4811; SSE4-NEXT: LBB6_3: ## %cond.store1812; SSE4-NEXT: extractps $1, %xmm0, 4(%rdi)813; SSE4-NEXT: testb $4, %al814; SSE4-NEXT: je LBB6_6815; SSE4-NEXT: LBB6_5: ## %cond.store3816; SSE4-NEXT: extractps $2, %xmm0, 8(%rdi)817; SSE4-NEXT: testb $8, %al818; SSE4-NEXT: je LBB6_8819; SSE4-NEXT: LBB6_7: ## %cond.store5820; SSE4-NEXT: extractps $3, %xmm0, 12(%rdi)821; SSE4-NEXT: testb $16, %al822; SSE4-NEXT: je LBB6_10823; SSE4-NEXT: LBB6_9: ## %cond.store7824; SSE4-NEXT: movss %xmm1, 16(%rdi)825; SSE4-NEXT: testb $32, %al826; SSE4-NEXT: je LBB6_12827; SSE4-NEXT: LBB6_11: ## %cond.store9828; SSE4-NEXT: extractps $1, %xmm1, 20(%rdi)829; SSE4-NEXT: testb $64, %al830; SSE4-NEXT: je LBB6_14831; SSE4-NEXT: LBB6_13: ## %cond.store11832; SSE4-NEXT: extractps $2, %xmm1, 24(%rdi)833; SSE4-NEXT: testb %al, %al834; SSE4-NEXT: jns LBB6_16835; SSE4-NEXT: LBB6_15: ## %cond.store13836; SSE4-NEXT: extractps $3, %xmm1, 28(%rdi)837; SSE4-NEXT: testl $256, %eax ## imm = 0x100838; SSE4-NEXT: je LBB6_18839; SSE4-NEXT: LBB6_17: ## %cond.store15840; SSE4-NEXT: movss %xmm2, 32(%rdi)841; SSE4-NEXT: testl $512, %eax ## imm = 0x200842; SSE4-NEXT: je LBB6_20843; SSE4-NEXT: LBB6_19: ## %cond.store17844; SSE4-NEXT: extractps $1, %xmm2, 36(%rdi)845; SSE4-NEXT: testl $1024, %eax ## imm = 0x400846; SSE4-NEXT: je LBB6_22847; SSE4-NEXT: LBB6_21: ## %cond.store19848; SSE4-NEXT: extractps $2, %xmm2, 40(%rdi)849; SSE4-NEXT: testl $2048, %eax ## imm = 0x800850; SSE4-NEXT: je LBB6_24851; SSE4-NEXT: LBB6_23: ## %cond.store21852; SSE4-NEXT: extractps $3, %xmm2, 44(%rdi)853; SSE4-NEXT: testl $4096, %eax ## imm = 0x1000854; SSE4-NEXT: je LBB6_26855; SSE4-NEXT: LBB6_25: ## %cond.store23856; SSE4-NEXT: movss %xmm3, 48(%rdi)857; SSE4-NEXT: testl $8192, %eax ## imm = 0x2000858; SSE4-NEXT: je LBB6_28859; SSE4-NEXT: LBB6_27: ## %cond.store25860; SSE4-NEXT: extractps $1, %xmm3, 52(%rdi)861; SSE4-NEXT: testl $16384, %eax ## imm = 0x4000862; SSE4-NEXT: je LBB6_30863; SSE4-NEXT: LBB6_29: ## %cond.store27864; SSE4-NEXT: extractps $2, %xmm3, 56(%rdi)865; SSE4-NEXT: testl $32768, %eax ## imm = 0x8000866; SSE4-NEXT: je LBB6_32867; SSE4-NEXT: LBB6_31: ## %cond.store29868; SSE4-NEXT: extractps $3, %xmm3, 60(%rdi)869; SSE4-NEXT: retq870;871; AVX1OR2-LABEL: store_v16f32_v16i32:872; AVX1OR2: ## %bb.0:873; AVX1OR2-NEXT: vmaskmovps %ymm1, %ymm5, 32(%rdi)874; AVX1OR2-NEXT: vmaskmovps %ymm0, %ymm4, (%rdi)875; AVX1OR2-NEXT: vzeroupper876; AVX1OR2-NEXT: retq877;878; AVX512F-LABEL: store_v16f32_v16i32:879; AVX512F: ## %bb.0:880; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1881; AVX512F-NEXT: vpcmpgtd %zmm2, %zmm1, %k1882; AVX512F-NEXT: vmovups %zmm0, (%rdi) {%k1}883; AVX512F-NEXT: vzeroupper884; AVX512F-NEXT: retq885;886; AVX512VLDQ-LABEL: store_v16f32_v16i32:887; AVX512VLDQ: ## %bb.0:888; AVX512VLDQ-NEXT: vpmovd2m %zmm2, %k1889; AVX512VLDQ-NEXT: vmovups %zmm0, (%rdi) {%k1}890; AVX512VLDQ-NEXT: vzeroupper891; AVX512VLDQ-NEXT: retq892;893; AVX512VLBW-LABEL: store_v16f32_v16i32:894; AVX512VLBW: ## %bb.0:895; AVX512VLBW-NEXT: vpxor %xmm1, %xmm1, %xmm1896; AVX512VLBW-NEXT: vpcmpgtd %zmm2, %zmm1, %k1897; AVX512VLBW-NEXT: vmovups %zmm0, (%rdi) {%k1}898; AVX512VLBW-NEXT: vzeroupper899; AVX512VLBW-NEXT: retq900;901; X86-AVX512-LABEL: store_v16f32_v16i32:902; X86-AVX512: ## %bb.0:903; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax904; X86-AVX512-NEXT: vpmovd2m %zmm2, %k1905; X86-AVX512-NEXT: vmovups %zmm0, (%eax) {%k1}906; X86-AVX512-NEXT: vzeroupper907; X86-AVX512-NEXT: retl908 %bool_mask = icmp slt <16 x i32> %mask, zeroinitializer909 call void @llvm.masked.store.v16f32.p0(<16 x float> %x, ptr %ptr, i32 1, <16 x i1> %bool_mask)910 ret void911}912 913;914; vXi64915;916 917define void @store_v2i64_v2i64(<2 x i64> %trigger, ptr %addr, <2 x i64> %val) nounwind {918; SSE2-LABEL: store_v2i64_v2i64:919; SSE2: ## %bb.0:920; SSE2-NEXT: movmskpd %xmm0, %eax921; SSE2-NEXT: testb $1, %al922; SSE2-NEXT: jne LBB7_1923; SSE2-NEXT: ## %bb.2: ## %else924; SSE2-NEXT: testb $2, %al925; SSE2-NEXT: jne LBB7_3926; SSE2-NEXT: LBB7_4: ## %else2927; SSE2-NEXT: retq928; SSE2-NEXT: LBB7_1: ## %cond.store929; SSE2-NEXT: movq %xmm1, (%rdi)930; SSE2-NEXT: testb $2, %al931; SSE2-NEXT: je LBB7_4932; SSE2-NEXT: LBB7_3: ## %cond.store1933; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]934; SSE2-NEXT: movq %xmm0, 8(%rdi)935; SSE2-NEXT: retq936;937; SSE4-LABEL: store_v2i64_v2i64:938; SSE4: ## %bb.0:939; SSE4-NEXT: movmskpd %xmm0, %eax940; SSE4-NEXT: testb $1, %al941; SSE4-NEXT: jne LBB7_1942; SSE4-NEXT: ## %bb.2: ## %else943; SSE4-NEXT: testb $2, %al944; SSE4-NEXT: jne LBB7_3945; SSE4-NEXT: LBB7_4: ## %else2946; SSE4-NEXT: retq947; SSE4-NEXT: LBB7_1: ## %cond.store948; SSE4-NEXT: movq %xmm1, (%rdi)949; SSE4-NEXT: testb $2, %al950; SSE4-NEXT: je LBB7_4951; SSE4-NEXT: LBB7_3: ## %cond.store1952; SSE4-NEXT: pextrq $1, %xmm1, 8(%rdi)953; SSE4-NEXT: retq954;955; AVX1-LABEL: store_v2i64_v2i64:956; AVX1: ## %bb.0:957; AVX1-NEXT: vmaskmovpd %xmm1, %xmm0, (%rdi)958; AVX1-NEXT: retq959;960; AVX2-LABEL: store_v2i64_v2i64:961; AVX2: ## %bb.0:962; AVX2-NEXT: vpmaskmovq %xmm1, %xmm0, (%rdi)963; AVX2-NEXT: retq964;965; AVX512F-LABEL: store_v2i64_v2i64:966; AVX512F: ## %bb.0:967; AVX512F-NEXT: ## kill: def $xmm1 killed $xmm1 def $zmm1968; AVX512F-NEXT: ## kill: def $xmm0 killed $xmm0 def $zmm0969; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2970; AVX512F-NEXT: vpcmpgtq %zmm0, %zmm2, %k0971; AVX512F-NEXT: kshiftlw $14, %k0, %k0972; AVX512F-NEXT: kshiftrw $14, %k0, %k1973; AVX512F-NEXT: vmovdqu64 %zmm1, (%rdi) {%k1}974; AVX512F-NEXT: vzeroupper975; AVX512F-NEXT: retq976;977; AVX512VLDQ-LABEL: store_v2i64_v2i64:978; AVX512VLDQ: ## %bb.0:979; AVX512VLDQ-NEXT: vpmovq2m %xmm0, %k1980; AVX512VLDQ-NEXT: vmovdqu64 %xmm1, (%rdi) {%k1}981; AVX512VLDQ-NEXT: retq982;983; AVX512VLBW-LABEL: store_v2i64_v2i64:984; AVX512VLBW: ## %bb.0:985; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2986; AVX512VLBW-NEXT: vpcmpgtq %xmm0, %xmm2, %k1987; AVX512VLBW-NEXT: vmovdqu64 %xmm1, (%rdi) {%k1}988; AVX512VLBW-NEXT: retq989;990; X86-AVX512-LABEL: store_v2i64_v2i64:991; X86-AVX512: ## %bb.0:992; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax993; X86-AVX512-NEXT: vpmovq2m %xmm0, %k1994; X86-AVX512-NEXT: vmovdqu64 %xmm1, (%eax) {%k1}995; X86-AVX512-NEXT: retl996 %mask = icmp slt <2 x i64> %trigger, zeroinitializer997 call void @llvm.masked.store.v2i64.p0(<2 x i64> %val, ptr %addr, i32 4, <2 x i1> %mask)998 ret void999}1000 1001define void @store_v4i64_v4i64(<4 x i64> %trigger, ptr %addr, <4 x i64> %val) nounwind {1002; SSE2-LABEL: store_v4i64_v4i64:1003; SSE2: ## %bb.0:1004; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,3]1005; SSE2-NEXT: movmskps %xmm0, %eax1006; SSE2-NEXT: testb $1, %al1007; SSE2-NEXT: jne LBB8_11008; SSE2-NEXT: ## %bb.2: ## %else1009; SSE2-NEXT: testb $2, %al1010; SSE2-NEXT: jne LBB8_31011; SSE2-NEXT: LBB8_4: ## %else21012; SSE2-NEXT: testb $4, %al1013; SSE2-NEXT: jne LBB8_51014; SSE2-NEXT: LBB8_6: ## %else41015; SSE2-NEXT: testb $8, %al1016; SSE2-NEXT: jne LBB8_71017; SSE2-NEXT: LBB8_8: ## %else61018; SSE2-NEXT: retq1019; SSE2-NEXT: LBB8_1: ## %cond.store1020; SSE2-NEXT: movq %xmm2, (%rdi)1021; SSE2-NEXT: testb $2, %al1022; SSE2-NEXT: je LBB8_41023; SSE2-NEXT: LBB8_3: ## %cond.store11024; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]1025; SSE2-NEXT: movq %xmm0, 8(%rdi)1026; SSE2-NEXT: testb $4, %al1027; SSE2-NEXT: je LBB8_61028; SSE2-NEXT: LBB8_5: ## %cond.store31029; SSE2-NEXT: movq %xmm3, 16(%rdi)1030; SSE2-NEXT: testb $8, %al1031; SSE2-NEXT: je LBB8_81032; SSE2-NEXT: LBB8_7: ## %cond.store51033; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]1034; SSE2-NEXT: movq %xmm0, 24(%rdi)1035; SSE2-NEXT: retq1036;1037; SSE4-LABEL: store_v4i64_v4i64:1038; SSE4: ## %bb.0:1039; SSE4-NEXT: packssdw %xmm1, %xmm01040; SSE4-NEXT: movmskps %xmm0, %eax1041; SSE4-NEXT: testb $1, %al1042; SSE4-NEXT: jne LBB8_11043; SSE4-NEXT: ## %bb.2: ## %else1044; SSE4-NEXT: testb $2, %al1045; SSE4-NEXT: jne LBB8_31046; SSE4-NEXT: LBB8_4: ## %else21047; SSE4-NEXT: testb $4, %al1048; SSE4-NEXT: jne LBB8_51049; SSE4-NEXT: LBB8_6: ## %else41050; SSE4-NEXT: testb $8, %al1051; SSE4-NEXT: jne LBB8_71052; SSE4-NEXT: LBB8_8: ## %else61053; SSE4-NEXT: retq1054; SSE4-NEXT: LBB8_1: ## %cond.store1055; SSE4-NEXT: movq %xmm2, (%rdi)1056; SSE4-NEXT: testb $2, %al1057; SSE4-NEXT: je LBB8_41058; SSE4-NEXT: LBB8_3: ## %cond.store11059; SSE4-NEXT: pextrq $1, %xmm2, 8(%rdi)1060; SSE4-NEXT: testb $4, %al1061; SSE4-NEXT: je LBB8_61062; SSE4-NEXT: LBB8_5: ## %cond.store31063; SSE4-NEXT: movq %xmm3, 16(%rdi)1064; SSE4-NEXT: testb $8, %al1065; SSE4-NEXT: je LBB8_81066; SSE4-NEXT: LBB8_7: ## %cond.store51067; SSE4-NEXT: pextrq $1, %xmm3, 24(%rdi)1068; SSE4-NEXT: retq1069;1070; AVX1-LABEL: store_v4i64_v4i64:1071; AVX1: ## %bb.0:1072; AVX1-NEXT: vmaskmovpd %ymm1, %ymm0, (%rdi)1073; AVX1-NEXT: vzeroupper1074; AVX1-NEXT: retq1075;1076; AVX2-LABEL: store_v4i64_v4i64:1077; AVX2: ## %bb.0:1078; AVX2-NEXT: vpmaskmovq %ymm1, %ymm0, (%rdi)1079; AVX2-NEXT: vzeroupper1080; AVX2-NEXT: retq1081;1082; AVX512F-LABEL: store_v4i64_v4i64:1083; AVX512F: ## %bb.0:1084; AVX512F-NEXT: ## kill: def $ymm1 killed $ymm1 def $zmm11085; AVX512F-NEXT: ## kill: def $ymm0 killed $ymm0 def $zmm01086; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm21087; AVX512F-NEXT: vpcmpgtq %zmm0, %zmm2, %k01088; AVX512F-NEXT: kshiftlw $12, %k0, %k01089; AVX512F-NEXT: kshiftrw $12, %k0, %k11090; AVX512F-NEXT: vmovdqu64 %zmm1, (%rdi) {%k1}1091; AVX512F-NEXT: vzeroupper1092; AVX512F-NEXT: retq1093;1094; AVX512VLDQ-LABEL: store_v4i64_v4i64:1095; AVX512VLDQ: ## %bb.0:1096; AVX512VLDQ-NEXT: vpmovq2m %ymm0, %k11097; AVX512VLDQ-NEXT: vmovdqu64 %ymm1, (%rdi) {%k1}1098; AVX512VLDQ-NEXT: vzeroupper1099; AVX512VLDQ-NEXT: retq1100;1101; AVX512VLBW-LABEL: store_v4i64_v4i64:1102; AVX512VLBW: ## %bb.0:1103; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm21104; AVX512VLBW-NEXT: vpcmpgtq %ymm0, %ymm2, %k11105; AVX512VLBW-NEXT: vmovdqu64 %ymm1, (%rdi) {%k1}1106; AVX512VLBW-NEXT: vzeroupper1107; AVX512VLBW-NEXT: retq1108;1109; X86-AVX512-LABEL: store_v4i64_v4i64:1110; X86-AVX512: ## %bb.0:1111; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax1112; X86-AVX512-NEXT: vpmovq2m %ymm0, %k11113; X86-AVX512-NEXT: vmovdqu64 %ymm1, (%eax) {%k1}1114; X86-AVX512-NEXT: vzeroupper1115; X86-AVX512-NEXT: retl1116 %mask = icmp slt <4 x i64> %trigger, zeroinitializer1117 call void @llvm.masked.store.v4i64.p0(<4 x i64> %val, ptr %addr, i32 4, <4 x i1> %mask)1118 ret void1119}1120 1121;1122; vXi321123;1124 1125define void @store_v1i32_v1i32(<1 x i32> %trigger, ptr %addr, <1 x i32> %val) nounwind {1126; SSE-LABEL: store_v1i32_v1i32:1127; SSE: ## %bb.0:1128; SSE-NEXT: testl %edi, %edi1129; SSE-NEXT: jne LBB9_21130; SSE-NEXT: ## %bb.1: ## %cond.store1131; SSE-NEXT: movl %edx, (%rsi)1132; SSE-NEXT: LBB9_2: ## %else1133; SSE-NEXT: retq1134;1135; AVX-LABEL: store_v1i32_v1i32:1136; AVX: ## %bb.0:1137; AVX-NEXT: testl %edi, %edi1138; AVX-NEXT: jne LBB9_21139; AVX-NEXT: ## %bb.1: ## %cond.store1140; AVX-NEXT: movl %edx, (%rsi)1141; AVX-NEXT: LBB9_2: ## %else1142; AVX-NEXT: retq1143;1144; X86-AVX512-LABEL: store_v1i32_v1i32:1145; X86-AVX512: ## %bb.0:1146; X86-AVX512-NEXT: cmpl $0, {{[0-9]+}}(%esp)1147; X86-AVX512-NEXT: jne LBB9_21148; X86-AVX512-NEXT: ## %bb.1: ## %cond.store1149; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax1150; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %ecx1151; X86-AVX512-NEXT: movl %eax, (%ecx)1152; X86-AVX512-NEXT: LBB9_2: ## %else1153; X86-AVX512-NEXT: retl1154 %mask = icmp eq <1 x i32> %trigger, zeroinitializer1155 call void @llvm.masked.store.v1i32.p0(<1 x i32> %val, ptr %addr, i32 4, <1 x i1> %mask)1156 ret void1157}1158 1159define void @store_v2i32_v2i32(<2 x i32> %trigger, ptr %addr, <2 x i32> %val) nounwind {1160; SSE2-LABEL: store_v2i32_v2i32:1161; SSE2: ## %bb.0:1162; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]1163; SSE2-NEXT: pxor %xmm2, %xmm21164; SSE2-NEXT: pcmpeqd %xmm0, %xmm21165; SSE2-NEXT: movmskpd %xmm2, %eax1166; SSE2-NEXT: testb $1, %al1167; SSE2-NEXT: jne LBB10_11168; SSE2-NEXT: ## %bb.2: ## %else1169; SSE2-NEXT: testb $2, %al1170; SSE2-NEXT: jne LBB10_31171; SSE2-NEXT: LBB10_4: ## %else21172; SSE2-NEXT: retq1173; SSE2-NEXT: LBB10_1: ## %cond.store1174; SSE2-NEXT: movd %xmm1, (%rdi)1175; SSE2-NEXT: testb $2, %al1176; SSE2-NEXT: je LBB10_41177; SSE2-NEXT: LBB10_3: ## %cond.store11178; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]1179; SSE2-NEXT: movd %xmm0, 4(%rdi)1180; SSE2-NEXT: retq1181;1182; SSE4-LABEL: store_v2i32_v2i32:1183; SSE4: ## %bb.0:1184; SSE4-NEXT: pxor %xmm2, %xmm21185; SSE4-NEXT: pcmpeqd %xmm0, %xmm21186; SSE4-NEXT: pmovsxdq %xmm2, %xmm01187; SSE4-NEXT: movmskpd %xmm0, %eax1188; SSE4-NEXT: testb $1, %al1189; SSE4-NEXT: jne LBB10_11190; SSE4-NEXT: ## %bb.2: ## %else1191; SSE4-NEXT: testb $2, %al1192; SSE4-NEXT: jne LBB10_31193; SSE4-NEXT: LBB10_4: ## %else21194; SSE4-NEXT: retq1195; SSE4-NEXT: LBB10_1: ## %cond.store1196; SSE4-NEXT: movss %xmm1, (%rdi)1197; SSE4-NEXT: testb $2, %al1198; SSE4-NEXT: je LBB10_41199; SSE4-NEXT: LBB10_3: ## %cond.store11200; SSE4-NEXT: extractps $1, %xmm1, 4(%rdi)1201; SSE4-NEXT: retq1202;1203; AVX1-LABEL: store_v2i32_v2i32:1204; AVX1: ## %bb.0:1205; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm21206; AVX1-NEXT: vpcmpeqd %xmm2, %xmm0, %xmm01207; AVX1-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero1208; AVX1-NEXT: vmaskmovps %xmm1, %xmm0, (%rdi)1209; AVX1-NEXT: retq1210;1211; AVX2-LABEL: store_v2i32_v2i32:1212; AVX2: ## %bb.0:1213; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm21214; AVX2-NEXT: vpcmpeqd %xmm2, %xmm0, %xmm01215; AVX2-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero1216; AVX2-NEXT: vpmaskmovd %xmm1, %xmm0, (%rdi)1217; AVX2-NEXT: retq1218;1219; AVX512F-LABEL: store_v2i32_v2i32:1220; AVX512F: ## %bb.0:1221; AVX512F-NEXT: ## kill: def $xmm1 killed $xmm1 def $zmm11222; AVX512F-NEXT: ## kill: def $xmm0 killed $xmm0 def $zmm01223; AVX512F-NEXT: vptestnmd %zmm0, %zmm0, %k01224; AVX512F-NEXT: kshiftlw $14, %k0, %k01225; AVX512F-NEXT: kshiftrw $14, %k0, %k11226; AVX512F-NEXT: vmovdqu32 %zmm1, (%rdi) {%k1}1227; AVX512F-NEXT: vzeroupper1228; AVX512F-NEXT: retq1229;1230; AVX512VLDQ-LABEL: store_v2i32_v2i32:1231; AVX512VLDQ: ## %bb.0:1232; AVX512VLDQ-NEXT: vptestnmd %xmm0, %xmm0, %k01233; AVX512VLDQ-NEXT: kshiftlb $6, %k0, %k01234; AVX512VLDQ-NEXT: kshiftrb $6, %k0, %k11235; AVX512VLDQ-NEXT: vmovdqu32 %xmm1, (%rdi) {%k1}1236; AVX512VLDQ-NEXT: retq1237;1238; AVX512VLBW-LABEL: store_v2i32_v2i32:1239; AVX512VLBW: ## %bb.0:1240; AVX512VLBW-NEXT: vptestnmd %xmm0, %xmm0, %k01241; AVX512VLBW-NEXT: kshiftlw $14, %k0, %k01242; AVX512VLBW-NEXT: kshiftrw $14, %k0, %k11243; AVX512VLBW-NEXT: vmovdqu32 %xmm1, (%rdi) {%k1}1244; AVX512VLBW-NEXT: retq1245;1246; X86-AVX512-LABEL: store_v2i32_v2i32:1247; X86-AVX512: ## %bb.0:1248; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax1249; X86-AVX512-NEXT: vptestnmd %xmm0, %xmm0, %k01250; X86-AVX512-NEXT: kshiftlb $6, %k0, %k01251; X86-AVX512-NEXT: kshiftrb $6, %k0, %k11252; X86-AVX512-NEXT: vmovdqu32 %xmm1, (%eax) {%k1}1253; X86-AVX512-NEXT: retl1254 %mask = icmp eq <2 x i32> %trigger, zeroinitializer1255 call void @llvm.masked.store.v2i32.p0(<2 x i32> %val, ptr %addr, i32 4, <2 x i1> %mask)1256 ret void1257}1258 1259define void @store_v4i32_v4i32(<4 x i32> %trigger, ptr %addr, <4 x i32> %val) nounwind {1260; SSE2-LABEL: store_v4i32_v4i32:1261; SSE2: ## %bb.0:1262; SSE2-NEXT: pxor %xmm2, %xmm21263; SSE2-NEXT: pcmpeqd %xmm0, %xmm21264; SSE2-NEXT: movmskps %xmm2, %eax1265; SSE2-NEXT: testb $1, %al1266; SSE2-NEXT: jne LBB11_11267; SSE2-NEXT: ## %bb.2: ## %else1268; SSE2-NEXT: testb $2, %al1269; SSE2-NEXT: jne LBB11_31270; SSE2-NEXT: LBB11_4: ## %else21271; SSE2-NEXT: testb $4, %al1272; SSE2-NEXT: jne LBB11_51273; SSE2-NEXT: LBB11_6: ## %else41274; SSE2-NEXT: testb $8, %al1275; SSE2-NEXT: jne LBB11_71276; SSE2-NEXT: LBB11_8: ## %else61277; SSE2-NEXT: retq1278; SSE2-NEXT: LBB11_1: ## %cond.store1279; SSE2-NEXT: movd %xmm1, (%rdi)1280; SSE2-NEXT: testb $2, %al1281; SSE2-NEXT: je LBB11_41282; SSE2-NEXT: LBB11_3: ## %cond.store11283; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]1284; SSE2-NEXT: movd %xmm0, 4(%rdi)1285; SSE2-NEXT: testb $4, %al1286; SSE2-NEXT: je LBB11_61287; SSE2-NEXT: LBB11_5: ## %cond.store31288; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]1289; SSE2-NEXT: movd %xmm0, 8(%rdi)1290; SSE2-NEXT: testb $8, %al1291; SSE2-NEXT: je LBB11_81292; SSE2-NEXT: LBB11_7: ## %cond.store51293; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]1294; SSE2-NEXT: movd %xmm0, 12(%rdi)1295; SSE2-NEXT: retq1296;1297; SSE4-LABEL: store_v4i32_v4i32:1298; SSE4: ## %bb.0:1299; SSE4-NEXT: pxor %xmm2, %xmm21300; SSE4-NEXT: pcmpeqd %xmm0, %xmm21301; SSE4-NEXT: movmskps %xmm2, %eax1302; SSE4-NEXT: testb $1, %al1303; SSE4-NEXT: jne LBB11_11304; SSE4-NEXT: ## %bb.2: ## %else1305; SSE4-NEXT: testb $2, %al1306; SSE4-NEXT: jne LBB11_31307; SSE4-NEXT: LBB11_4: ## %else21308; SSE4-NEXT: testb $4, %al1309; SSE4-NEXT: jne LBB11_51310; SSE4-NEXT: LBB11_6: ## %else41311; SSE4-NEXT: testb $8, %al1312; SSE4-NEXT: jne LBB11_71313; SSE4-NEXT: LBB11_8: ## %else61314; SSE4-NEXT: retq1315; SSE4-NEXT: LBB11_1: ## %cond.store1316; SSE4-NEXT: movss %xmm1, (%rdi)1317; SSE4-NEXT: testb $2, %al1318; SSE4-NEXT: je LBB11_41319; SSE4-NEXT: LBB11_3: ## %cond.store11320; SSE4-NEXT: extractps $1, %xmm1, 4(%rdi)1321; SSE4-NEXT: testb $4, %al1322; SSE4-NEXT: je LBB11_61323; SSE4-NEXT: LBB11_5: ## %cond.store31324; SSE4-NEXT: extractps $2, %xmm1, 8(%rdi)1325; SSE4-NEXT: testb $8, %al1326; SSE4-NEXT: je LBB11_81327; SSE4-NEXT: LBB11_7: ## %cond.store51328; SSE4-NEXT: extractps $3, %xmm1, 12(%rdi)1329; SSE4-NEXT: retq1330;1331; AVX1-LABEL: store_v4i32_v4i32:1332; AVX1: ## %bb.0:1333; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm21334; AVX1-NEXT: vpcmpeqd %xmm2, %xmm0, %xmm01335; AVX1-NEXT: vmaskmovps %xmm1, %xmm0, (%rdi)1336; AVX1-NEXT: retq1337;1338; AVX2-LABEL: store_v4i32_v4i32:1339; AVX2: ## %bb.0:1340; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm21341; AVX2-NEXT: vpcmpeqd %xmm2, %xmm0, %xmm01342; AVX2-NEXT: vpmaskmovd %xmm1, %xmm0, (%rdi)1343; AVX2-NEXT: retq1344;1345; AVX512F-LABEL: store_v4i32_v4i32:1346; AVX512F: ## %bb.0:1347; AVX512F-NEXT: ## kill: def $xmm1 killed $xmm1 def $zmm11348; AVX512F-NEXT: ## kill: def $xmm0 killed $xmm0 def $zmm01349; AVX512F-NEXT: vptestnmd %zmm0, %zmm0, %k01350; AVX512F-NEXT: kshiftlw $12, %k0, %k01351; AVX512F-NEXT: kshiftrw $12, %k0, %k11352; AVX512F-NEXT: vmovdqu32 %zmm1, (%rdi) {%k1}1353; AVX512F-NEXT: vzeroupper1354; AVX512F-NEXT: retq1355;1356; AVX512VL-LABEL: store_v4i32_v4i32:1357; AVX512VL: ## %bb.0:1358; AVX512VL-NEXT: vptestnmd %xmm0, %xmm0, %k11359; AVX512VL-NEXT: vmovdqu32 %xmm1, (%rdi) {%k1}1360; AVX512VL-NEXT: retq1361;1362; X86-AVX512-LABEL: store_v4i32_v4i32:1363; X86-AVX512: ## %bb.0:1364; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax1365; X86-AVX512-NEXT: vptestnmd %xmm0, %xmm0, %k11366; X86-AVX512-NEXT: vmovdqu32 %xmm1, (%eax) {%k1}1367; X86-AVX512-NEXT: retl1368 %mask = icmp eq <4 x i32> %trigger, zeroinitializer1369 call void @llvm.masked.store.v4i32.p0(<4 x i32> %val, ptr %addr, i32 4, <4 x i1> %mask)1370 ret void1371}1372 1373define void @store_v8i32_v8i32(<8 x i32> %trigger, ptr %addr, <8 x i32> %val) nounwind {1374; SSE2-LABEL: store_v8i32_v8i32:1375; SSE2: ## %bb.0:1376; SSE2-NEXT: pxor %xmm4, %xmm41377; SSE2-NEXT: pcmpeqd %xmm4, %xmm11378; SSE2-NEXT: pcmpeqd %xmm4, %xmm01379; SSE2-NEXT: packssdw %xmm1, %xmm01380; SSE2-NEXT: packsswb %xmm0, %xmm01381; SSE2-NEXT: pmovmskb %xmm0, %eax1382; SSE2-NEXT: testb $1, %al1383; SSE2-NEXT: jne LBB12_11384; SSE2-NEXT: ## %bb.2: ## %else1385; SSE2-NEXT: testb $2, %al1386; SSE2-NEXT: jne LBB12_31387; SSE2-NEXT: LBB12_4: ## %else21388; SSE2-NEXT: testb $4, %al1389; SSE2-NEXT: jne LBB12_51390; SSE2-NEXT: LBB12_6: ## %else41391; SSE2-NEXT: testb $8, %al1392; SSE2-NEXT: jne LBB12_71393; SSE2-NEXT: LBB12_8: ## %else61394; SSE2-NEXT: testb $16, %al1395; SSE2-NEXT: jne LBB12_91396; SSE2-NEXT: LBB12_10: ## %else81397; SSE2-NEXT: testb $32, %al1398; SSE2-NEXT: jne LBB12_111399; SSE2-NEXT: LBB12_12: ## %else101400; SSE2-NEXT: testb $64, %al1401; SSE2-NEXT: jne LBB12_131402; SSE2-NEXT: LBB12_14: ## %else121403; SSE2-NEXT: testb $-128, %al1404; SSE2-NEXT: jne LBB12_151405; SSE2-NEXT: LBB12_16: ## %else141406; SSE2-NEXT: retq1407; SSE2-NEXT: LBB12_1: ## %cond.store1408; SSE2-NEXT: movd %xmm2, (%rdi)1409; SSE2-NEXT: testb $2, %al1410; SSE2-NEXT: je LBB12_41411; SSE2-NEXT: LBB12_3: ## %cond.store11412; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]1413; SSE2-NEXT: movd %xmm0, 4(%rdi)1414; SSE2-NEXT: testb $4, %al1415; SSE2-NEXT: je LBB12_61416; SSE2-NEXT: LBB12_5: ## %cond.store31417; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]1418; SSE2-NEXT: movd %xmm0, 8(%rdi)1419; SSE2-NEXT: testb $8, %al1420; SSE2-NEXT: je LBB12_81421; SSE2-NEXT: LBB12_7: ## %cond.store51422; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]1423; SSE2-NEXT: movd %xmm0, 12(%rdi)1424; SSE2-NEXT: testb $16, %al1425; SSE2-NEXT: je LBB12_101426; SSE2-NEXT: LBB12_9: ## %cond.store71427; SSE2-NEXT: movd %xmm3, 16(%rdi)1428; SSE2-NEXT: testb $32, %al1429; SSE2-NEXT: je LBB12_121430; SSE2-NEXT: LBB12_11: ## %cond.store91431; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]1432; SSE2-NEXT: movd %xmm0, 20(%rdi)1433; SSE2-NEXT: testb $64, %al1434; SSE2-NEXT: je LBB12_141435; SSE2-NEXT: LBB12_13: ## %cond.store111436; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]1437; SSE2-NEXT: movd %xmm0, 24(%rdi)1438; SSE2-NEXT: testb $-128, %al1439; SSE2-NEXT: je LBB12_161440; SSE2-NEXT: LBB12_15: ## %cond.store131441; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[3,3,3,3]1442; SSE2-NEXT: movd %xmm0, 28(%rdi)1443; SSE2-NEXT: retq1444;1445; SSE4-LABEL: store_v8i32_v8i32:1446; SSE4: ## %bb.0:1447; SSE4-NEXT: pxor %xmm4, %xmm41448; SSE4-NEXT: pcmpeqd %xmm4, %xmm11449; SSE4-NEXT: pcmpeqd %xmm4, %xmm01450; SSE4-NEXT: packssdw %xmm1, %xmm01451; SSE4-NEXT: packsswb %xmm0, %xmm01452; SSE4-NEXT: pmovmskb %xmm0, %eax1453; SSE4-NEXT: testb $1, %al1454; SSE4-NEXT: jne LBB12_11455; SSE4-NEXT: ## %bb.2: ## %else1456; SSE4-NEXT: testb $2, %al1457; SSE4-NEXT: jne LBB12_31458; SSE4-NEXT: LBB12_4: ## %else21459; SSE4-NEXT: testb $4, %al1460; SSE4-NEXT: jne LBB12_51461; SSE4-NEXT: LBB12_6: ## %else41462; SSE4-NEXT: testb $8, %al1463; SSE4-NEXT: jne LBB12_71464; SSE4-NEXT: LBB12_8: ## %else61465; SSE4-NEXT: testb $16, %al1466; SSE4-NEXT: jne LBB12_91467; SSE4-NEXT: LBB12_10: ## %else81468; SSE4-NEXT: testb $32, %al1469; SSE4-NEXT: jne LBB12_111470; SSE4-NEXT: LBB12_12: ## %else101471; SSE4-NEXT: testb $64, %al1472; SSE4-NEXT: jne LBB12_131473; SSE4-NEXT: LBB12_14: ## %else121474; SSE4-NEXT: testb $-128, %al1475; SSE4-NEXT: jne LBB12_151476; SSE4-NEXT: LBB12_16: ## %else141477; SSE4-NEXT: retq1478; SSE4-NEXT: LBB12_1: ## %cond.store1479; SSE4-NEXT: movss %xmm2, (%rdi)1480; SSE4-NEXT: testb $2, %al1481; SSE4-NEXT: je LBB12_41482; SSE4-NEXT: LBB12_3: ## %cond.store11483; SSE4-NEXT: extractps $1, %xmm2, 4(%rdi)1484; SSE4-NEXT: testb $4, %al1485; SSE4-NEXT: je LBB12_61486; SSE4-NEXT: LBB12_5: ## %cond.store31487; SSE4-NEXT: extractps $2, %xmm2, 8(%rdi)1488; SSE4-NEXT: testb $8, %al1489; SSE4-NEXT: je LBB12_81490; SSE4-NEXT: LBB12_7: ## %cond.store51491; SSE4-NEXT: extractps $3, %xmm2, 12(%rdi)1492; SSE4-NEXT: testb $16, %al1493; SSE4-NEXT: je LBB12_101494; SSE4-NEXT: LBB12_9: ## %cond.store71495; SSE4-NEXT: movss %xmm3, 16(%rdi)1496; SSE4-NEXT: testb $32, %al1497; SSE4-NEXT: je LBB12_121498; SSE4-NEXT: LBB12_11: ## %cond.store91499; SSE4-NEXT: extractps $1, %xmm3, 20(%rdi)1500; SSE4-NEXT: testb $64, %al1501; SSE4-NEXT: je LBB12_141502; SSE4-NEXT: LBB12_13: ## %cond.store111503; SSE4-NEXT: extractps $2, %xmm3, 24(%rdi)1504; SSE4-NEXT: testb $-128, %al1505; SSE4-NEXT: je LBB12_161506; SSE4-NEXT: LBB12_15: ## %cond.store131507; SSE4-NEXT: extractps $3, %xmm3, 28(%rdi)1508; SSE4-NEXT: retq1509;1510; AVX1-LABEL: store_v8i32_v8i32:1511; AVX1: ## %bb.0:1512; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm21513; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm31514; AVX1-NEXT: vpcmpeqd %xmm3, %xmm2, %xmm21515; AVX1-NEXT: vpcmpeqd %xmm3, %xmm0, %xmm01516; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm01517; AVX1-NEXT: vmaskmovps %ymm1, %ymm0, (%rdi)1518; AVX1-NEXT: vzeroupper1519; AVX1-NEXT: retq1520;1521; AVX2-LABEL: store_v8i32_v8i32:1522; AVX2: ## %bb.0:1523; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm21524; AVX2-NEXT: vpcmpeqd %ymm2, %ymm0, %ymm01525; AVX2-NEXT: vpmaskmovd %ymm1, %ymm0, (%rdi)1526; AVX2-NEXT: vzeroupper1527; AVX2-NEXT: retq1528;1529; AVX512F-LABEL: store_v8i32_v8i32:1530; AVX512F: ## %bb.0:1531; AVX512F-NEXT: ## kill: def $ymm1 killed $ymm1 def $zmm11532; AVX512F-NEXT: ## kill: def $ymm0 killed $ymm0 def $zmm01533; AVX512F-NEXT: vptestnmd %zmm0, %zmm0, %k01534; AVX512F-NEXT: kshiftlw $8, %k0, %k01535; AVX512F-NEXT: kshiftrw $8, %k0, %k11536; AVX512F-NEXT: vmovdqu32 %zmm1, (%rdi) {%k1}1537; AVX512F-NEXT: vzeroupper1538; AVX512F-NEXT: retq1539;1540; AVX512VL-LABEL: store_v8i32_v8i32:1541; AVX512VL: ## %bb.0:1542; AVX512VL-NEXT: vptestnmd %ymm0, %ymm0, %k11543; AVX512VL-NEXT: vmovdqu32 %ymm1, (%rdi) {%k1}1544; AVX512VL-NEXT: vzeroupper1545; AVX512VL-NEXT: retq1546;1547; X86-AVX512-LABEL: store_v8i32_v8i32:1548; X86-AVX512: ## %bb.0:1549; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax1550; X86-AVX512-NEXT: vptestnmd %ymm0, %ymm0, %k11551; X86-AVX512-NEXT: vmovdqu32 %ymm1, (%eax) {%k1}1552; X86-AVX512-NEXT: vzeroupper1553; X86-AVX512-NEXT: retl1554 %mask = icmp eq <8 x i32> %trigger, zeroinitializer1555 call void @llvm.masked.store.v8i32.p0(<8 x i32> %val, ptr %addr, i32 4, <8 x i1> %mask)1556 ret void1557}1558 1559;1560; vXi161561;1562 1563define void @store_v8i16_v8i16(<8 x i16> %trigger, ptr %addr, <8 x i16> %val) nounwind {1564; SSE2-LABEL: store_v8i16_v8i16:1565; SSE2: ## %bb.0:1566; SSE2-NEXT: pxor %xmm2, %xmm21567; SSE2-NEXT: pcmpeqw %xmm0, %xmm21568; SSE2-NEXT: packsswb %xmm2, %xmm21569; SSE2-NEXT: pmovmskb %xmm2, %eax1570; SSE2-NEXT: testb $1, %al1571; SSE2-NEXT: jne LBB13_11572; SSE2-NEXT: ## %bb.2: ## %else1573; SSE2-NEXT: testb $2, %al1574; SSE2-NEXT: jne LBB13_31575; SSE2-NEXT: LBB13_4: ## %else21576; SSE2-NEXT: testb $4, %al1577; SSE2-NEXT: jne LBB13_51578; SSE2-NEXT: LBB13_6: ## %else41579; SSE2-NEXT: testb $8, %al1580; SSE2-NEXT: jne LBB13_71581; SSE2-NEXT: LBB13_8: ## %else61582; SSE2-NEXT: testb $16, %al1583; SSE2-NEXT: jne LBB13_91584; SSE2-NEXT: LBB13_10: ## %else81585; SSE2-NEXT: testb $32, %al1586; SSE2-NEXT: jne LBB13_111587; SSE2-NEXT: LBB13_12: ## %else101588; SSE2-NEXT: testb $64, %al1589; SSE2-NEXT: jne LBB13_131590; SSE2-NEXT: LBB13_14: ## %else121591; SSE2-NEXT: testb $-128, %al1592; SSE2-NEXT: jne LBB13_151593; SSE2-NEXT: LBB13_16: ## %else141594; SSE2-NEXT: retq1595; SSE2-NEXT: LBB13_1: ## %cond.store1596; SSE2-NEXT: movd %xmm1, %ecx1597; SSE2-NEXT: movw %cx, (%rdi)1598; SSE2-NEXT: testb $2, %al1599; SSE2-NEXT: je LBB13_41600; SSE2-NEXT: LBB13_3: ## %cond.store11601; SSE2-NEXT: pextrw $1, %xmm1, %ecx1602; SSE2-NEXT: movw %cx, 2(%rdi)1603; SSE2-NEXT: testb $4, %al1604; SSE2-NEXT: je LBB13_61605; SSE2-NEXT: LBB13_5: ## %cond.store31606; SSE2-NEXT: pextrw $2, %xmm1, %ecx1607; SSE2-NEXT: movw %cx, 4(%rdi)1608; SSE2-NEXT: testb $8, %al1609; SSE2-NEXT: je LBB13_81610; SSE2-NEXT: LBB13_7: ## %cond.store51611; SSE2-NEXT: pextrw $3, %xmm1, %ecx1612; SSE2-NEXT: movw %cx, 6(%rdi)1613; SSE2-NEXT: testb $16, %al1614; SSE2-NEXT: je LBB13_101615; SSE2-NEXT: LBB13_9: ## %cond.store71616; SSE2-NEXT: pextrw $4, %xmm1, %ecx1617; SSE2-NEXT: movw %cx, 8(%rdi)1618; SSE2-NEXT: testb $32, %al1619; SSE2-NEXT: je LBB13_121620; SSE2-NEXT: LBB13_11: ## %cond.store91621; SSE2-NEXT: pextrw $5, %xmm1, %ecx1622; SSE2-NEXT: movw %cx, 10(%rdi)1623; SSE2-NEXT: testb $64, %al1624; SSE2-NEXT: je LBB13_141625; SSE2-NEXT: LBB13_13: ## %cond.store111626; SSE2-NEXT: pextrw $6, %xmm1, %ecx1627; SSE2-NEXT: movw %cx, 12(%rdi)1628; SSE2-NEXT: testb $-128, %al1629; SSE2-NEXT: je LBB13_161630; SSE2-NEXT: LBB13_15: ## %cond.store131631; SSE2-NEXT: pextrw $7, %xmm1, %eax1632; SSE2-NEXT: movw %ax, 14(%rdi)1633; SSE2-NEXT: retq1634;1635; SSE4-LABEL: store_v8i16_v8i16:1636; SSE4: ## %bb.0:1637; SSE4-NEXT: pxor %xmm2, %xmm21638; SSE4-NEXT: pcmpeqw %xmm0, %xmm21639; SSE4-NEXT: packsswb %xmm2, %xmm21640; SSE4-NEXT: pmovmskb %xmm2, %eax1641; SSE4-NEXT: testb $1, %al1642; SSE4-NEXT: jne LBB13_11643; SSE4-NEXT: ## %bb.2: ## %else1644; SSE4-NEXT: testb $2, %al1645; SSE4-NEXT: jne LBB13_31646; SSE4-NEXT: LBB13_4: ## %else21647; SSE4-NEXT: testb $4, %al1648; SSE4-NEXT: jne LBB13_51649; SSE4-NEXT: LBB13_6: ## %else41650; SSE4-NEXT: testb $8, %al1651; SSE4-NEXT: jne LBB13_71652; SSE4-NEXT: LBB13_8: ## %else61653; SSE4-NEXT: testb $16, %al1654; SSE4-NEXT: jne LBB13_91655; SSE4-NEXT: LBB13_10: ## %else81656; SSE4-NEXT: testb $32, %al1657; SSE4-NEXT: jne LBB13_111658; SSE4-NEXT: LBB13_12: ## %else101659; SSE4-NEXT: testb $64, %al1660; SSE4-NEXT: jne LBB13_131661; SSE4-NEXT: LBB13_14: ## %else121662; SSE4-NEXT: testb $-128, %al1663; SSE4-NEXT: jne LBB13_151664; SSE4-NEXT: LBB13_16: ## %else141665; SSE4-NEXT: retq1666; SSE4-NEXT: LBB13_1: ## %cond.store1667; SSE4-NEXT: pextrw $0, %xmm1, (%rdi)1668; SSE4-NEXT: testb $2, %al1669; SSE4-NEXT: je LBB13_41670; SSE4-NEXT: LBB13_3: ## %cond.store11671; SSE4-NEXT: pextrw $1, %xmm1, 2(%rdi)1672; SSE4-NEXT: testb $4, %al1673; SSE4-NEXT: je LBB13_61674; SSE4-NEXT: LBB13_5: ## %cond.store31675; SSE4-NEXT: pextrw $2, %xmm1, 4(%rdi)1676; SSE4-NEXT: testb $8, %al1677; SSE4-NEXT: je LBB13_81678; SSE4-NEXT: LBB13_7: ## %cond.store51679; SSE4-NEXT: pextrw $3, %xmm1, 6(%rdi)1680; SSE4-NEXT: testb $16, %al1681; SSE4-NEXT: je LBB13_101682; SSE4-NEXT: LBB13_9: ## %cond.store71683; SSE4-NEXT: pextrw $4, %xmm1, 8(%rdi)1684; SSE4-NEXT: testb $32, %al1685; SSE4-NEXT: je LBB13_121686; SSE4-NEXT: LBB13_11: ## %cond.store91687; SSE4-NEXT: pextrw $5, %xmm1, 10(%rdi)1688; SSE4-NEXT: testb $64, %al1689; SSE4-NEXT: je LBB13_141690; SSE4-NEXT: LBB13_13: ## %cond.store111691; SSE4-NEXT: pextrw $6, %xmm1, 12(%rdi)1692; SSE4-NEXT: testb $-128, %al1693; SSE4-NEXT: je LBB13_161694; SSE4-NEXT: LBB13_15: ## %cond.store131695; SSE4-NEXT: pextrw $7, %xmm1, 14(%rdi)1696; SSE4-NEXT: retq1697;1698; AVX1OR2-LABEL: store_v8i16_v8i16:1699; AVX1OR2: ## %bb.0:1700; AVX1OR2-NEXT: vpxor %xmm2, %xmm2, %xmm21701; AVX1OR2-NEXT: vpcmpeqw %xmm2, %xmm0, %xmm01702; AVX1OR2-NEXT: vpacksswb %xmm0, %xmm0, %xmm01703; AVX1OR2-NEXT: vpmovmskb %xmm0, %eax1704; AVX1OR2-NEXT: testb $1, %al1705; AVX1OR2-NEXT: jne LBB13_11706; AVX1OR2-NEXT: ## %bb.2: ## %else1707; AVX1OR2-NEXT: testb $2, %al1708; AVX1OR2-NEXT: jne LBB13_31709; AVX1OR2-NEXT: LBB13_4: ## %else21710; AVX1OR2-NEXT: testb $4, %al1711; AVX1OR2-NEXT: jne LBB13_51712; AVX1OR2-NEXT: LBB13_6: ## %else41713; AVX1OR2-NEXT: testb $8, %al1714; AVX1OR2-NEXT: jne LBB13_71715; AVX1OR2-NEXT: LBB13_8: ## %else61716; AVX1OR2-NEXT: testb $16, %al1717; AVX1OR2-NEXT: jne LBB13_91718; AVX1OR2-NEXT: LBB13_10: ## %else81719; AVX1OR2-NEXT: testb $32, %al1720; AVX1OR2-NEXT: jne LBB13_111721; AVX1OR2-NEXT: LBB13_12: ## %else101722; AVX1OR2-NEXT: testb $64, %al1723; AVX1OR2-NEXT: jne LBB13_131724; AVX1OR2-NEXT: LBB13_14: ## %else121725; AVX1OR2-NEXT: testb $-128, %al1726; AVX1OR2-NEXT: jne LBB13_151727; AVX1OR2-NEXT: LBB13_16: ## %else141728; AVX1OR2-NEXT: retq1729; AVX1OR2-NEXT: LBB13_1: ## %cond.store1730; AVX1OR2-NEXT: vpextrw $0, %xmm1, (%rdi)1731; AVX1OR2-NEXT: testb $2, %al1732; AVX1OR2-NEXT: je LBB13_41733; AVX1OR2-NEXT: LBB13_3: ## %cond.store11734; AVX1OR2-NEXT: vpextrw $1, %xmm1, 2(%rdi)1735; AVX1OR2-NEXT: testb $4, %al1736; AVX1OR2-NEXT: je LBB13_61737; AVX1OR2-NEXT: LBB13_5: ## %cond.store31738; AVX1OR2-NEXT: vpextrw $2, %xmm1, 4(%rdi)1739; AVX1OR2-NEXT: testb $8, %al1740; AVX1OR2-NEXT: je LBB13_81741; AVX1OR2-NEXT: LBB13_7: ## %cond.store51742; AVX1OR2-NEXT: vpextrw $3, %xmm1, 6(%rdi)1743; AVX1OR2-NEXT: testb $16, %al1744; AVX1OR2-NEXT: je LBB13_101745; AVX1OR2-NEXT: LBB13_9: ## %cond.store71746; AVX1OR2-NEXT: vpextrw $4, %xmm1, 8(%rdi)1747; AVX1OR2-NEXT: testb $32, %al1748; AVX1OR2-NEXT: je LBB13_121749; AVX1OR2-NEXT: LBB13_11: ## %cond.store91750; AVX1OR2-NEXT: vpextrw $5, %xmm1, 10(%rdi)1751; AVX1OR2-NEXT: testb $64, %al1752; AVX1OR2-NEXT: je LBB13_141753; AVX1OR2-NEXT: LBB13_13: ## %cond.store111754; AVX1OR2-NEXT: vpextrw $6, %xmm1, 12(%rdi)1755; AVX1OR2-NEXT: testb $-128, %al1756; AVX1OR2-NEXT: je LBB13_161757; AVX1OR2-NEXT: LBB13_15: ## %cond.store131758; AVX1OR2-NEXT: vpextrw $7, %xmm1, 14(%rdi)1759; AVX1OR2-NEXT: retq1760;1761; AVX512F-LABEL: store_v8i16_v8i16:1762; AVX512F: ## %bb.0:1763; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm21764; AVX512F-NEXT: vpcmpeqw %xmm2, %xmm0, %xmm01765; AVX512F-NEXT: vpmovsxwq %xmm0, %zmm01766; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k01767; AVX512F-NEXT: kmovw %k0, %eax1768; AVX512F-NEXT: testb $1, %al1769; AVX512F-NEXT: jne LBB13_11770; AVX512F-NEXT: ## %bb.2: ## %else1771; AVX512F-NEXT: testb $2, %al1772; AVX512F-NEXT: jne LBB13_31773; AVX512F-NEXT: LBB13_4: ## %else21774; AVX512F-NEXT: testb $4, %al1775; AVX512F-NEXT: jne LBB13_51776; AVX512F-NEXT: LBB13_6: ## %else41777; AVX512F-NEXT: testb $8, %al1778; AVX512F-NEXT: jne LBB13_71779; AVX512F-NEXT: LBB13_8: ## %else61780; AVX512F-NEXT: testb $16, %al1781; AVX512F-NEXT: jne LBB13_91782; AVX512F-NEXT: LBB13_10: ## %else81783; AVX512F-NEXT: testb $32, %al1784; AVX512F-NEXT: jne LBB13_111785; AVX512F-NEXT: LBB13_12: ## %else101786; AVX512F-NEXT: testb $64, %al1787; AVX512F-NEXT: jne LBB13_131788; AVX512F-NEXT: LBB13_14: ## %else121789; AVX512F-NEXT: testb $-128, %al1790; AVX512F-NEXT: jne LBB13_151791; AVX512F-NEXT: LBB13_16: ## %else141792; AVX512F-NEXT: vzeroupper1793; AVX512F-NEXT: retq1794; AVX512F-NEXT: LBB13_1: ## %cond.store1795; AVX512F-NEXT: vpextrw $0, %xmm1, (%rdi)1796; AVX512F-NEXT: testb $2, %al1797; AVX512F-NEXT: je LBB13_41798; AVX512F-NEXT: LBB13_3: ## %cond.store11799; AVX512F-NEXT: vpextrw $1, %xmm1, 2(%rdi)1800; AVX512F-NEXT: testb $4, %al1801; AVX512F-NEXT: je LBB13_61802; AVX512F-NEXT: LBB13_5: ## %cond.store31803; AVX512F-NEXT: vpextrw $2, %xmm1, 4(%rdi)1804; AVX512F-NEXT: testb $8, %al1805; AVX512F-NEXT: je LBB13_81806; AVX512F-NEXT: LBB13_7: ## %cond.store51807; AVX512F-NEXT: vpextrw $3, %xmm1, 6(%rdi)1808; AVX512F-NEXT: testb $16, %al1809; AVX512F-NEXT: je LBB13_101810; AVX512F-NEXT: LBB13_9: ## %cond.store71811; AVX512F-NEXT: vpextrw $4, %xmm1, 8(%rdi)1812; AVX512F-NEXT: testb $32, %al1813; AVX512F-NEXT: je LBB13_121814; AVX512F-NEXT: LBB13_11: ## %cond.store91815; AVX512F-NEXT: vpextrw $5, %xmm1, 10(%rdi)1816; AVX512F-NEXT: testb $64, %al1817; AVX512F-NEXT: je LBB13_141818; AVX512F-NEXT: LBB13_13: ## %cond.store111819; AVX512F-NEXT: vpextrw $6, %xmm1, 12(%rdi)1820; AVX512F-NEXT: testb $-128, %al1821; AVX512F-NEXT: je LBB13_161822; AVX512F-NEXT: LBB13_15: ## %cond.store131823; AVX512F-NEXT: vpextrw $7, %xmm1, 14(%rdi)1824; AVX512F-NEXT: vzeroupper1825; AVX512F-NEXT: retq1826;1827; AVX512VLDQ-LABEL: store_v8i16_v8i16:1828; AVX512VLDQ: ## %bb.0:1829; AVX512VLDQ-NEXT: vpxor %xmm2, %xmm2, %xmm21830; AVX512VLDQ-NEXT: vpcmpeqw %xmm2, %xmm0, %xmm01831; AVX512VLDQ-NEXT: vpmovsxwd %xmm0, %ymm01832; AVX512VLDQ-NEXT: vpmovd2m %ymm0, %k01833; AVX512VLDQ-NEXT: kmovw %k0, %eax1834; AVX512VLDQ-NEXT: testb $1, %al1835; AVX512VLDQ-NEXT: jne LBB13_11836; AVX512VLDQ-NEXT: ## %bb.2: ## %else1837; AVX512VLDQ-NEXT: testb $2, %al1838; AVX512VLDQ-NEXT: jne LBB13_31839; AVX512VLDQ-NEXT: LBB13_4: ## %else21840; AVX512VLDQ-NEXT: testb $4, %al1841; AVX512VLDQ-NEXT: jne LBB13_51842; AVX512VLDQ-NEXT: LBB13_6: ## %else41843; AVX512VLDQ-NEXT: testb $8, %al1844; AVX512VLDQ-NEXT: jne LBB13_71845; AVX512VLDQ-NEXT: LBB13_8: ## %else61846; AVX512VLDQ-NEXT: testb $16, %al1847; AVX512VLDQ-NEXT: jne LBB13_91848; AVX512VLDQ-NEXT: LBB13_10: ## %else81849; AVX512VLDQ-NEXT: testb $32, %al1850; AVX512VLDQ-NEXT: jne LBB13_111851; AVX512VLDQ-NEXT: LBB13_12: ## %else101852; AVX512VLDQ-NEXT: testb $64, %al1853; AVX512VLDQ-NEXT: jne LBB13_131854; AVX512VLDQ-NEXT: LBB13_14: ## %else121855; AVX512VLDQ-NEXT: testb $-128, %al1856; AVX512VLDQ-NEXT: jne LBB13_151857; AVX512VLDQ-NEXT: LBB13_16: ## %else141858; AVX512VLDQ-NEXT: vzeroupper1859; AVX512VLDQ-NEXT: retq1860; AVX512VLDQ-NEXT: LBB13_1: ## %cond.store1861; AVX512VLDQ-NEXT: vpextrw $0, %xmm1, (%rdi)1862; AVX512VLDQ-NEXT: testb $2, %al1863; AVX512VLDQ-NEXT: je LBB13_41864; AVX512VLDQ-NEXT: LBB13_3: ## %cond.store11865; AVX512VLDQ-NEXT: vpextrw $1, %xmm1, 2(%rdi)1866; AVX512VLDQ-NEXT: testb $4, %al1867; AVX512VLDQ-NEXT: je LBB13_61868; AVX512VLDQ-NEXT: LBB13_5: ## %cond.store31869; AVX512VLDQ-NEXT: vpextrw $2, %xmm1, 4(%rdi)1870; AVX512VLDQ-NEXT: testb $8, %al1871; AVX512VLDQ-NEXT: je LBB13_81872; AVX512VLDQ-NEXT: LBB13_7: ## %cond.store51873; AVX512VLDQ-NEXT: vpextrw $3, %xmm1, 6(%rdi)1874; AVX512VLDQ-NEXT: testb $16, %al1875; AVX512VLDQ-NEXT: je LBB13_101876; AVX512VLDQ-NEXT: LBB13_9: ## %cond.store71877; AVX512VLDQ-NEXT: vpextrw $4, %xmm1, 8(%rdi)1878; AVX512VLDQ-NEXT: testb $32, %al1879; AVX512VLDQ-NEXT: je LBB13_121880; AVX512VLDQ-NEXT: LBB13_11: ## %cond.store91881; AVX512VLDQ-NEXT: vpextrw $5, %xmm1, 10(%rdi)1882; AVX512VLDQ-NEXT: testb $64, %al1883; AVX512VLDQ-NEXT: je LBB13_141884; AVX512VLDQ-NEXT: LBB13_13: ## %cond.store111885; AVX512VLDQ-NEXT: vpextrw $6, %xmm1, 12(%rdi)1886; AVX512VLDQ-NEXT: testb $-128, %al1887; AVX512VLDQ-NEXT: je LBB13_161888; AVX512VLDQ-NEXT: LBB13_15: ## %cond.store131889; AVX512VLDQ-NEXT: vpextrw $7, %xmm1, 14(%rdi)1890; AVX512VLDQ-NEXT: vzeroupper1891; AVX512VLDQ-NEXT: retq1892;1893; AVX512VLBW-LABEL: store_v8i16_v8i16:1894; AVX512VLBW: ## %bb.0:1895; AVX512VLBW-NEXT: vptestnmw %xmm0, %xmm0, %k11896; AVX512VLBW-NEXT: vmovdqu16 %xmm1, (%rdi) {%k1}1897; AVX512VLBW-NEXT: retq1898;1899; X86-AVX512-LABEL: store_v8i16_v8i16:1900; X86-AVX512: ## %bb.0:1901; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax1902; X86-AVX512-NEXT: vptestnmw %xmm0, %xmm0, %k11903; X86-AVX512-NEXT: vmovdqu16 %xmm1, (%eax) {%k1}1904; X86-AVX512-NEXT: retl1905 %mask = icmp eq <8 x i16> %trigger, zeroinitializer1906 call void @llvm.masked.store.v8i16.p0(<8 x i16> %val, ptr %addr, i32 4, <8 x i1> %mask)1907 ret void1908}1909 1910define void @store_v16i16_v16i16(<16 x i16> %trigger, ptr %addr, <16 x i16> %val) nounwind {1911; SSE2-LABEL: store_v16i16_v16i16:1912; SSE2: ## %bb.0:1913; SSE2-NEXT: pxor %xmm4, %xmm41914; SSE2-NEXT: pcmpeqw %xmm4, %xmm11915; SSE2-NEXT: pcmpeqw %xmm4, %xmm01916; SSE2-NEXT: packsswb %xmm1, %xmm01917; SSE2-NEXT: pmovmskb %xmm0, %eax1918; SSE2-NEXT: testb $1, %al1919; SSE2-NEXT: jne LBB14_11920; SSE2-NEXT: ## %bb.2: ## %else1921; SSE2-NEXT: testb $2, %al1922; SSE2-NEXT: jne LBB14_31923; SSE2-NEXT: LBB14_4: ## %else21924; SSE2-NEXT: testb $4, %al1925; SSE2-NEXT: jne LBB14_51926; SSE2-NEXT: LBB14_6: ## %else41927; SSE2-NEXT: testb $8, %al1928; SSE2-NEXT: jne LBB14_71929; SSE2-NEXT: LBB14_8: ## %else61930; SSE2-NEXT: testb $16, %al1931; SSE2-NEXT: jne LBB14_91932; SSE2-NEXT: LBB14_10: ## %else81933; SSE2-NEXT: testb $32, %al1934; SSE2-NEXT: jne LBB14_111935; SSE2-NEXT: LBB14_12: ## %else101936; SSE2-NEXT: testb $64, %al1937; SSE2-NEXT: jne LBB14_131938; SSE2-NEXT: LBB14_14: ## %else121939; SSE2-NEXT: testb %al, %al1940; SSE2-NEXT: js LBB14_151941; SSE2-NEXT: LBB14_16: ## %else141942; SSE2-NEXT: testl $256, %eax ## imm = 0x1001943; SSE2-NEXT: jne LBB14_171944; SSE2-NEXT: LBB14_18: ## %else161945; SSE2-NEXT: testl $512, %eax ## imm = 0x2001946; SSE2-NEXT: jne LBB14_191947; SSE2-NEXT: LBB14_20: ## %else181948; SSE2-NEXT: testl $1024, %eax ## imm = 0x4001949; SSE2-NEXT: jne LBB14_211950; SSE2-NEXT: LBB14_22: ## %else201951; SSE2-NEXT: testl $2048, %eax ## imm = 0x8001952; SSE2-NEXT: jne LBB14_231953; SSE2-NEXT: LBB14_24: ## %else221954; SSE2-NEXT: testl $4096, %eax ## imm = 0x10001955; SSE2-NEXT: jne LBB14_251956; SSE2-NEXT: LBB14_26: ## %else241957; SSE2-NEXT: testl $8192, %eax ## imm = 0x20001958; SSE2-NEXT: jne LBB14_271959; SSE2-NEXT: LBB14_28: ## %else261960; SSE2-NEXT: testl $16384, %eax ## imm = 0x40001961; SSE2-NEXT: jne LBB14_291962; SSE2-NEXT: LBB14_30: ## %else281963; SSE2-NEXT: testl $32768, %eax ## imm = 0x80001964; SSE2-NEXT: jne LBB14_311965; SSE2-NEXT: LBB14_32: ## %else301966; SSE2-NEXT: retq1967; SSE2-NEXT: LBB14_1: ## %cond.store1968; SSE2-NEXT: movd %xmm2, %ecx1969; SSE2-NEXT: movw %cx, (%rdi)1970; SSE2-NEXT: testb $2, %al1971; SSE2-NEXT: je LBB14_41972; SSE2-NEXT: LBB14_3: ## %cond.store11973; SSE2-NEXT: pextrw $1, %xmm2, %ecx1974; SSE2-NEXT: movw %cx, 2(%rdi)1975; SSE2-NEXT: testb $4, %al1976; SSE2-NEXT: je LBB14_61977; SSE2-NEXT: LBB14_5: ## %cond.store31978; SSE2-NEXT: pextrw $2, %xmm2, %ecx1979; SSE2-NEXT: movw %cx, 4(%rdi)1980; SSE2-NEXT: testb $8, %al1981; SSE2-NEXT: je LBB14_81982; SSE2-NEXT: LBB14_7: ## %cond.store51983; SSE2-NEXT: pextrw $3, %xmm2, %ecx1984; SSE2-NEXT: movw %cx, 6(%rdi)1985; SSE2-NEXT: testb $16, %al1986; SSE2-NEXT: je LBB14_101987; SSE2-NEXT: LBB14_9: ## %cond.store71988; SSE2-NEXT: pextrw $4, %xmm2, %ecx1989; SSE2-NEXT: movw %cx, 8(%rdi)1990; SSE2-NEXT: testb $32, %al1991; SSE2-NEXT: je LBB14_121992; SSE2-NEXT: LBB14_11: ## %cond.store91993; SSE2-NEXT: pextrw $5, %xmm2, %ecx1994; SSE2-NEXT: movw %cx, 10(%rdi)1995; SSE2-NEXT: testb $64, %al1996; SSE2-NEXT: je LBB14_141997; SSE2-NEXT: LBB14_13: ## %cond.store111998; SSE2-NEXT: pextrw $6, %xmm2, %ecx1999; SSE2-NEXT: movw %cx, 12(%rdi)2000; SSE2-NEXT: testb %al, %al2001; SSE2-NEXT: jns LBB14_162002; SSE2-NEXT: LBB14_15: ## %cond.store132003; SSE2-NEXT: pextrw $7, %xmm2, %ecx2004; SSE2-NEXT: movw %cx, 14(%rdi)2005; SSE2-NEXT: testl $256, %eax ## imm = 0x1002006; SSE2-NEXT: je LBB14_182007; SSE2-NEXT: LBB14_17: ## %cond.store152008; SSE2-NEXT: movd %xmm3, %ecx2009; SSE2-NEXT: movw %cx, 16(%rdi)2010; SSE2-NEXT: testl $512, %eax ## imm = 0x2002011; SSE2-NEXT: je LBB14_202012; SSE2-NEXT: LBB14_19: ## %cond.store172013; SSE2-NEXT: pextrw $1, %xmm3, %ecx2014; SSE2-NEXT: movw %cx, 18(%rdi)2015; SSE2-NEXT: testl $1024, %eax ## imm = 0x4002016; SSE2-NEXT: je LBB14_222017; SSE2-NEXT: LBB14_21: ## %cond.store192018; SSE2-NEXT: pextrw $2, %xmm3, %ecx2019; SSE2-NEXT: movw %cx, 20(%rdi)2020; SSE2-NEXT: testl $2048, %eax ## imm = 0x8002021; SSE2-NEXT: je LBB14_242022; SSE2-NEXT: LBB14_23: ## %cond.store212023; SSE2-NEXT: pextrw $3, %xmm3, %ecx2024; SSE2-NEXT: movw %cx, 22(%rdi)2025; SSE2-NEXT: testl $4096, %eax ## imm = 0x10002026; SSE2-NEXT: je LBB14_262027; SSE2-NEXT: LBB14_25: ## %cond.store232028; SSE2-NEXT: pextrw $4, %xmm3, %ecx2029; SSE2-NEXT: movw %cx, 24(%rdi)2030; SSE2-NEXT: testl $8192, %eax ## imm = 0x20002031; SSE2-NEXT: je LBB14_282032; SSE2-NEXT: LBB14_27: ## %cond.store252033; SSE2-NEXT: pextrw $5, %xmm3, %ecx2034; SSE2-NEXT: movw %cx, 26(%rdi)2035; SSE2-NEXT: testl $16384, %eax ## imm = 0x40002036; SSE2-NEXT: je LBB14_302037; SSE2-NEXT: LBB14_29: ## %cond.store272038; SSE2-NEXT: pextrw $6, %xmm3, %ecx2039; SSE2-NEXT: movw %cx, 28(%rdi)2040; SSE2-NEXT: testl $32768, %eax ## imm = 0x80002041; SSE2-NEXT: je LBB14_322042; SSE2-NEXT: LBB14_31: ## %cond.store292043; SSE2-NEXT: pextrw $7, %xmm3, %eax2044; SSE2-NEXT: movw %ax, 30(%rdi)2045; SSE2-NEXT: retq2046;2047; SSE4-LABEL: store_v16i16_v16i16:2048; SSE4: ## %bb.0:2049; SSE4-NEXT: pxor %xmm4, %xmm42050; SSE4-NEXT: pcmpeqw %xmm4, %xmm12051; SSE4-NEXT: pcmpeqw %xmm4, %xmm02052; SSE4-NEXT: packsswb %xmm1, %xmm02053; SSE4-NEXT: pmovmskb %xmm0, %eax2054; SSE4-NEXT: testb $1, %al2055; SSE4-NEXT: jne LBB14_12056; SSE4-NEXT: ## %bb.2: ## %else2057; SSE4-NEXT: testb $2, %al2058; SSE4-NEXT: jne LBB14_32059; SSE4-NEXT: LBB14_4: ## %else22060; SSE4-NEXT: testb $4, %al2061; SSE4-NEXT: jne LBB14_52062; SSE4-NEXT: LBB14_6: ## %else42063; SSE4-NEXT: testb $8, %al2064; SSE4-NEXT: jne LBB14_72065; SSE4-NEXT: LBB14_8: ## %else62066; SSE4-NEXT: testb $16, %al2067; SSE4-NEXT: jne LBB14_92068; SSE4-NEXT: LBB14_10: ## %else82069; SSE4-NEXT: testb $32, %al2070; SSE4-NEXT: jne LBB14_112071; SSE4-NEXT: LBB14_12: ## %else102072; SSE4-NEXT: testb $64, %al2073; SSE4-NEXT: jne LBB14_132074; SSE4-NEXT: LBB14_14: ## %else122075; SSE4-NEXT: testb %al, %al2076; SSE4-NEXT: js LBB14_152077; SSE4-NEXT: LBB14_16: ## %else142078; SSE4-NEXT: testl $256, %eax ## imm = 0x1002079; SSE4-NEXT: jne LBB14_172080; SSE4-NEXT: LBB14_18: ## %else162081; SSE4-NEXT: testl $512, %eax ## imm = 0x2002082; SSE4-NEXT: jne LBB14_192083; SSE4-NEXT: LBB14_20: ## %else182084; SSE4-NEXT: testl $1024, %eax ## imm = 0x4002085; SSE4-NEXT: jne LBB14_212086; SSE4-NEXT: LBB14_22: ## %else202087; SSE4-NEXT: testl $2048, %eax ## imm = 0x8002088; SSE4-NEXT: jne LBB14_232089; SSE4-NEXT: LBB14_24: ## %else222090; SSE4-NEXT: testl $4096, %eax ## imm = 0x10002091; SSE4-NEXT: jne LBB14_252092; SSE4-NEXT: LBB14_26: ## %else242093; SSE4-NEXT: testl $8192, %eax ## imm = 0x20002094; SSE4-NEXT: jne LBB14_272095; SSE4-NEXT: LBB14_28: ## %else262096; SSE4-NEXT: testl $16384, %eax ## imm = 0x40002097; SSE4-NEXT: jne LBB14_292098; SSE4-NEXT: LBB14_30: ## %else282099; SSE4-NEXT: testl $32768, %eax ## imm = 0x80002100; SSE4-NEXT: jne LBB14_312101; SSE4-NEXT: LBB14_32: ## %else302102; SSE4-NEXT: retq2103; SSE4-NEXT: LBB14_1: ## %cond.store2104; SSE4-NEXT: pextrw $0, %xmm2, (%rdi)2105; SSE4-NEXT: testb $2, %al2106; SSE4-NEXT: je LBB14_42107; SSE4-NEXT: LBB14_3: ## %cond.store12108; SSE4-NEXT: pextrw $1, %xmm2, 2(%rdi)2109; SSE4-NEXT: testb $4, %al2110; SSE4-NEXT: je LBB14_62111; SSE4-NEXT: LBB14_5: ## %cond.store32112; SSE4-NEXT: pextrw $2, %xmm2, 4(%rdi)2113; SSE4-NEXT: testb $8, %al2114; SSE4-NEXT: je LBB14_82115; SSE4-NEXT: LBB14_7: ## %cond.store52116; SSE4-NEXT: pextrw $3, %xmm2, 6(%rdi)2117; SSE4-NEXT: testb $16, %al2118; SSE4-NEXT: je LBB14_102119; SSE4-NEXT: LBB14_9: ## %cond.store72120; SSE4-NEXT: pextrw $4, %xmm2, 8(%rdi)2121; SSE4-NEXT: testb $32, %al2122; SSE4-NEXT: je LBB14_122123; SSE4-NEXT: LBB14_11: ## %cond.store92124; SSE4-NEXT: pextrw $5, %xmm2, 10(%rdi)2125; SSE4-NEXT: testb $64, %al2126; SSE4-NEXT: je LBB14_142127; SSE4-NEXT: LBB14_13: ## %cond.store112128; SSE4-NEXT: pextrw $6, %xmm2, 12(%rdi)2129; SSE4-NEXT: testb %al, %al2130; SSE4-NEXT: jns LBB14_162131; SSE4-NEXT: LBB14_15: ## %cond.store132132; SSE4-NEXT: pextrw $7, %xmm2, 14(%rdi)2133; SSE4-NEXT: testl $256, %eax ## imm = 0x1002134; SSE4-NEXT: je LBB14_182135; SSE4-NEXT: LBB14_17: ## %cond.store152136; SSE4-NEXT: pextrw $0, %xmm3, 16(%rdi)2137; SSE4-NEXT: testl $512, %eax ## imm = 0x2002138; SSE4-NEXT: je LBB14_202139; SSE4-NEXT: LBB14_19: ## %cond.store172140; SSE4-NEXT: pextrw $1, %xmm3, 18(%rdi)2141; SSE4-NEXT: testl $1024, %eax ## imm = 0x4002142; SSE4-NEXT: je LBB14_222143; SSE4-NEXT: LBB14_21: ## %cond.store192144; SSE4-NEXT: pextrw $2, %xmm3, 20(%rdi)2145; SSE4-NEXT: testl $2048, %eax ## imm = 0x8002146; SSE4-NEXT: je LBB14_242147; SSE4-NEXT: LBB14_23: ## %cond.store212148; SSE4-NEXT: pextrw $3, %xmm3, 22(%rdi)2149; SSE4-NEXT: testl $4096, %eax ## imm = 0x10002150; SSE4-NEXT: je LBB14_262151; SSE4-NEXT: LBB14_25: ## %cond.store232152; SSE4-NEXT: pextrw $4, %xmm3, 24(%rdi)2153; SSE4-NEXT: testl $8192, %eax ## imm = 0x20002154; SSE4-NEXT: je LBB14_282155; SSE4-NEXT: LBB14_27: ## %cond.store252156; SSE4-NEXT: pextrw $5, %xmm3, 26(%rdi)2157; SSE4-NEXT: testl $16384, %eax ## imm = 0x40002158; SSE4-NEXT: je LBB14_302159; SSE4-NEXT: LBB14_29: ## %cond.store272160; SSE4-NEXT: pextrw $6, %xmm3, 28(%rdi)2161; SSE4-NEXT: testl $32768, %eax ## imm = 0x80002162; SSE4-NEXT: je LBB14_322163; SSE4-NEXT: LBB14_31: ## %cond.store292164; SSE4-NEXT: pextrw $7, %xmm3, 30(%rdi)2165; SSE4-NEXT: retq2166;2167; AVX1-LABEL: store_v16i16_v16i16:2168; AVX1: ## %bb.0:2169; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm22170; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm32171; AVX1-NEXT: vpcmpeqw %xmm3, %xmm2, %xmm22172; AVX1-NEXT: vpcmpeqw %xmm3, %xmm0, %xmm02173; AVX1-NEXT: vpacksswb %xmm2, %xmm0, %xmm02174; AVX1-NEXT: vpmovmskb %xmm0, %eax2175; AVX1-NEXT: testb $1, %al2176; AVX1-NEXT: jne LBB14_12177; AVX1-NEXT: ## %bb.2: ## %else2178; AVX1-NEXT: testb $2, %al2179; AVX1-NEXT: jne LBB14_32180; AVX1-NEXT: LBB14_4: ## %else22181; AVX1-NEXT: testb $4, %al2182; AVX1-NEXT: jne LBB14_52183; AVX1-NEXT: LBB14_6: ## %else42184; AVX1-NEXT: testb $8, %al2185; AVX1-NEXT: jne LBB14_72186; AVX1-NEXT: LBB14_8: ## %else62187; AVX1-NEXT: testb $16, %al2188; AVX1-NEXT: jne LBB14_92189; AVX1-NEXT: LBB14_10: ## %else82190; AVX1-NEXT: testb $32, %al2191; AVX1-NEXT: jne LBB14_112192; AVX1-NEXT: LBB14_12: ## %else102193; AVX1-NEXT: testb $64, %al2194; AVX1-NEXT: jne LBB14_132195; AVX1-NEXT: LBB14_14: ## %else122196; AVX1-NEXT: testb %al, %al2197; AVX1-NEXT: jns LBB14_162198; AVX1-NEXT: LBB14_15: ## %cond.store132199; AVX1-NEXT: vpextrw $7, %xmm1, 14(%rdi)2200; AVX1-NEXT: LBB14_16: ## %else142201; AVX1-NEXT: testl $256, %eax ## imm = 0x1002202; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm02203; AVX1-NEXT: jne LBB14_172204; AVX1-NEXT: ## %bb.18: ## %else162205; AVX1-NEXT: testl $512, %eax ## imm = 0x2002206; AVX1-NEXT: jne LBB14_192207; AVX1-NEXT: LBB14_20: ## %else182208; AVX1-NEXT: testl $1024, %eax ## imm = 0x4002209; AVX1-NEXT: jne LBB14_212210; AVX1-NEXT: LBB14_22: ## %else202211; AVX1-NEXT: testl $2048, %eax ## imm = 0x8002212; AVX1-NEXT: jne LBB14_232213; AVX1-NEXT: LBB14_24: ## %else222214; AVX1-NEXT: testl $4096, %eax ## imm = 0x10002215; AVX1-NEXT: jne LBB14_252216; AVX1-NEXT: LBB14_26: ## %else242217; AVX1-NEXT: testl $8192, %eax ## imm = 0x20002218; AVX1-NEXT: jne LBB14_272219; AVX1-NEXT: LBB14_28: ## %else262220; AVX1-NEXT: testl $16384, %eax ## imm = 0x40002221; AVX1-NEXT: jne LBB14_292222; AVX1-NEXT: LBB14_30: ## %else282223; AVX1-NEXT: testl $32768, %eax ## imm = 0x80002224; AVX1-NEXT: jne LBB14_312225; AVX1-NEXT: LBB14_32: ## %else302226; AVX1-NEXT: vzeroupper2227; AVX1-NEXT: retq2228; AVX1-NEXT: LBB14_1: ## %cond.store2229; AVX1-NEXT: vpextrw $0, %xmm1, (%rdi)2230; AVX1-NEXT: testb $2, %al2231; AVX1-NEXT: je LBB14_42232; AVX1-NEXT: LBB14_3: ## %cond.store12233; AVX1-NEXT: vpextrw $1, %xmm1, 2(%rdi)2234; AVX1-NEXT: testb $4, %al2235; AVX1-NEXT: je LBB14_62236; AVX1-NEXT: LBB14_5: ## %cond.store32237; AVX1-NEXT: vpextrw $2, %xmm1, 4(%rdi)2238; AVX1-NEXT: testb $8, %al2239; AVX1-NEXT: je LBB14_82240; AVX1-NEXT: LBB14_7: ## %cond.store52241; AVX1-NEXT: vpextrw $3, %xmm1, 6(%rdi)2242; AVX1-NEXT: testb $16, %al2243; AVX1-NEXT: je LBB14_102244; AVX1-NEXT: LBB14_9: ## %cond.store72245; AVX1-NEXT: vpextrw $4, %xmm1, 8(%rdi)2246; AVX1-NEXT: testb $32, %al2247; AVX1-NEXT: je LBB14_122248; AVX1-NEXT: LBB14_11: ## %cond.store92249; AVX1-NEXT: vpextrw $5, %xmm1, 10(%rdi)2250; AVX1-NEXT: testb $64, %al2251; AVX1-NEXT: je LBB14_142252; AVX1-NEXT: LBB14_13: ## %cond.store112253; AVX1-NEXT: vpextrw $6, %xmm1, 12(%rdi)2254; AVX1-NEXT: testb %al, %al2255; AVX1-NEXT: js LBB14_152256; AVX1-NEXT: jmp LBB14_162257; AVX1-NEXT: LBB14_17: ## %cond.store152258; AVX1-NEXT: vpextrw $0, %xmm0, 16(%rdi)2259; AVX1-NEXT: testl $512, %eax ## imm = 0x2002260; AVX1-NEXT: je LBB14_202261; AVX1-NEXT: LBB14_19: ## %cond.store172262; AVX1-NEXT: vpextrw $1, %xmm0, 18(%rdi)2263; AVX1-NEXT: testl $1024, %eax ## imm = 0x4002264; AVX1-NEXT: je LBB14_222265; AVX1-NEXT: LBB14_21: ## %cond.store192266; AVX1-NEXT: vpextrw $2, %xmm0, 20(%rdi)2267; AVX1-NEXT: testl $2048, %eax ## imm = 0x8002268; AVX1-NEXT: je LBB14_242269; AVX1-NEXT: LBB14_23: ## %cond.store212270; AVX1-NEXT: vpextrw $3, %xmm0, 22(%rdi)2271; AVX1-NEXT: testl $4096, %eax ## imm = 0x10002272; AVX1-NEXT: je LBB14_262273; AVX1-NEXT: LBB14_25: ## %cond.store232274; AVX1-NEXT: vpextrw $4, %xmm0, 24(%rdi)2275; AVX1-NEXT: testl $8192, %eax ## imm = 0x20002276; AVX1-NEXT: je LBB14_282277; AVX1-NEXT: LBB14_27: ## %cond.store252278; AVX1-NEXT: vpextrw $5, %xmm0, 26(%rdi)2279; AVX1-NEXT: testl $16384, %eax ## imm = 0x40002280; AVX1-NEXT: je LBB14_302281; AVX1-NEXT: LBB14_29: ## %cond.store272282; AVX1-NEXT: vpextrw $6, %xmm0, 28(%rdi)2283; AVX1-NEXT: testl $32768, %eax ## imm = 0x80002284; AVX1-NEXT: je LBB14_322285; AVX1-NEXT: LBB14_31: ## %cond.store292286; AVX1-NEXT: vpextrw $7, %xmm0, 30(%rdi)2287; AVX1-NEXT: vzeroupper2288; AVX1-NEXT: retq2289;2290; AVX2-LABEL: store_v16i16_v16i16:2291; AVX2: ## %bb.0:2292; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm22293; AVX2-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm02294; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm22295; AVX2-NEXT: vpacksswb %xmm2, %xmm0, %xmm02296; AVX2-NEXT: vpmovmskb %xmm0, %eax2297; AVX2-NEXT: testb $1, %al2298; AVX2-NEXT: jne LBB14_12299; AVX2-NEXT: ## %bb.2: ## %else2300; AVX2-NEXT: testb $2, %al2301; AVX2-NEXT: jne LBB14_32302; AVX2-NEXT: LBB14_4: ## %else22303; AVX2-NEXT: testb $4, %al2304; AVX2-NEXT: jne LBB14_52305; AVX2-NEXT: LBB14_6: ## %else42306; AVX2-NEXT: testb $8, %al2307; AVX2-NEXT: jne LBB14_72308; AVX2-NEXT: LBB14_8: ## %else62309; AVX2-NEXT: testb $16, %al2310; AVX2-NEXT: jne LBB14_92311; AVX2-NEXT: LBB14_10: ## %else82312; AVX2-NEXT: testb $32, %al2313; AVX2-NEXT: jne LBB14_112314; AVX2-NEXT: LBB14_12: ## %else102315; AVX2-NEXT: testb $64, %al2316; AVX2-NEXT: jne LBB14_132317; AVX2-NEXT: LBB14_14: ## %else122318; AVX2-NEXT: testb %al, %al2319; AVX2-NEXT: jns LBB14_162320; AVX2-NEXT: LBB14_15: ## %cond.store132321; AVX2-NEXT: vpextrw $7, %xmm1, 14(%rdi)2322; AVX2-NEXT: LBB14_16: ## %else142323; AVX2-NEXT: testl $256, %eax ## imm = 0x1002324; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm02325; AVX2-NEXT: jne LBB14_172326; AVX2-NEXT: ## %bb.18: ## %else162327; AVX2-NEXT: testl $512, %eax ## imm = 0x2002328; AVX2-NEXT: jne LBB14_192329; AVX2-NEXT: LBB14_20: ## %else182330; AVX2-NEXT: testl $1024, %eax ## imm = 0x4002331; AVX2-NEXT: jne LBB14_212332; AVX2-NEXT: LBB14_22: ## %else202333; AVX2-NEXT: testl $2048, %eax ## imm = 0x8002334; AVX2-NEXT: jne LBB14_232335; AVX2-NEXT: LBB14_24: ## %else222336; AVX2-NEXT: testl $4096, %eax ## imm = 0x10002337; AVX2-NEXT: jne LBB14_252338; AVX2-NEXT: LBB14_26: ## %else242339; AVX2-NEXT: testl $8192, %eax ## imm = 0x20002340; AVX2-NEXT: jne LBB14_272341; AVX2-NEXT: LBB14_28: ## %else262342; AVX2-NEXT: testl $16384, %eax ## imm = 0x40002343; AVX2-NEXT: jne LBB14_292344; AVX2-NEXT: LBB14_30: ## %else282345; AVX2-NEXT: testl $32768, %eax ## imm = 0x80002346; AVX2-NEXT: jne LBB14_312347; AVX2-NEXT: LBB14_32: ## %else302348; AVX2-NEXT: vzeroupper2349; AVX2-NEXT: retq2350; AVX2-NEXT: LBB14_1: ## %cond.store2351; AVX2-NEXT: vpextrw $0, %xmm1, (%rdi)2352; AVX2-NEXT: testb $2, %al2353; AVX2-NEXT: je LBB14_42354; AVX2-NEXT: LBB14_3: ## %cond.store12355; AVX2-NEXT: vpextrw $1, %xmm1, 2(%rdi)2356; AVX2-NEXT: testb $4, %al2357; AVX2-NEXT: je LBB14_62358; AVX2-NEXT: LBB14_5: ## %cond.store32359; AVX2-NEXT: vpextrw $2, %xmm1, 4(%rdi)2360; AVX2-NEXT: testb $8, %al2361; AVX2-NEXT: je LBB14_82362; AVX2-NEXT: LBB14_7: ## %cond.store52363; AVX2-NEXT: vpextrw $3, %xmm1, 6(%rdi)2364; AVX2-NEXT: testb $16, %al2365; AVX2-NEXT: je LBB14_102366; AVX2-NEXT: LBB14_9: ## %cond.store72367; AVX2-NEXT: vpextrw $4, %xmm1, 8(%rdi)2368; AVX2-NEXT: testb $32, %al2369; AVX2-NEXT: je LBB14_122370; AVX2-NEXT: LBB14_11: ## %cond.store92371; AVX2-NEXT: vpextrw $5, %xmm1, 10(%rdi)2372; AVX2-NEXT: testb $64, %al2373; AVX2-NEXT: je LBB14_142374; AVX2-NEXT: LBB14_13: ## %cond.store112375; AVX2-NEXT: vpextrw $6, %xmm1, 12(%rdi)2376; AVX2-NEXT: testb %al, %al2377; AVX2-NEXT: js LBB14_152378; AVX2-NEXT: jmp LBB14_162379; AVX2-NEXT: LBB14_17: ## %cond.store152380; AVX2-NEXT: vpextrw $0, %xmm0, 16(%rdi)2381; AVX2-NEXT: testl $512, %eax ## imm = 0x2002382; AVX2-NEXT: je LBB14_202383; AVX2-NEXT: LBB14_19: ## %cond.store172384; AVX2-NEXT: vpextrw $1, %xmm0, 18(%rdi)2385; AVX2-NEXT: testl $1024, %eax ## imm = 0x4002386; AVX2-NEXT: je LBB14_222387; AVX2-NEXT: LBB14_21: ## %cond.store192388; AVX2-NEXT: vpextrw $2, %xmm0, 20(%rdi)2389; AVX2-NEXT: testl $2048, %eax ## imm = 0x8002390; AVX2-NEXT: je LBB14_242391; AVX2-NEXT: LBB14_23: ## %cond.store212392; AVX2-NEXT: vpextrw $3, %xmm0, 22(%rdi)2393; AVX2-NEXT: testl $4096, %eax ## imm = 0x10002394; AVX2-NEXT: je LBB14_262395; AVX2-NEXT: LBB14_25: ## %cond.store232396; AVX2-NEXT: vpextrw $4, %xmm0, 24(%rdi)2397; AVX2-NEXT: testl $8192, %eax ## imm = 0x20002398; AVX2-NEXT: je LBB14_282399; AVX2-NEXT: LBB14_27: ## %cond.store252400; AVX2-NEXT: vpextrw $5, %xmm0, 26(%rdi)2401; AVX2-NEXT: testl $16384, %eax ## imm = 0x40002402; AVX2-NEXT: je LBB14_302403; AVX2-NEXT: LBB14_29: ## %cond.store272404; AVX2-NEXT: vpextrw $6, %xmm0, 28(%rdi)2405; AVX2-NEXT: testl $32768, %eax ## imm = 0x80002406; AVX2-NEXT: je LBB14_322407; AVX2-NEXT: LBB14_31: ## %cond.store292408; AVX2-NEXT: vpextrw $7, %xmm0, 30(%rdi)2409; AVX2-NEXT: vzeroupper2410; AVX2-NEXT: retq2411;2412; AVX512F-LABEL: store_v16i16_v16i16:2413; AVX512F: ## %bb.0:2414; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm22415; AVX512F-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm02416; AVX512F-NEXT: vpmovsxwd %ymm0, %zmm02417; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k02418; AVX512F-NEXT: kmovw %k0, %eax2419; AVX512F-NEXT: testb $1, %al2420; AVX512F-NEXT: jne LBB14_12421; AVX512F-NEXT: ## %bb.2: ## %else2422; AVX512F-NEXT: testb $2, %al2423; AVX512F-NEXT: jne LBB14_32424; AVX512F-NEXT: LBB14_4: ## %else22425; AVX512F-NEXT: testb $4, %al2426; AVX512F-NEXT: jne LBB14_52427; AVX512F-NEXT: LBB14_6: ## %else42428; AVX512F-NEXT: testb $8, %al2429; AVX512F-NEXT: jne LBB14_72430; AVX512F-NEXT: LBB14_8: ## %else62431; AVX512F-NEXT: testb $16, %al2432; AVX512F-NEXT: jne LBB14_92433; AVX512F-NEXT: LBB14_10: ## %else82434; AVX512F-NEXT: testb $32, %al2435; AVX512F-NEXT: jne LBB14_112436; AVX512F-NEXT: LBB14_12: ## %else102437; AVX512F-NEXT: testb $64, %al2438; AVX512F-NEXT: jne LBB14_132439; AVX512F-NEXT: LBB14_14: ## %else122440; AVX512F-NEXT: testb %al, %al2441; AVX512F-NEXT: jns LBB14_162442; AVX512F-NEXT: LBB14_15: ## %cond.store132443; AVX512F-NEXT: vpextrw $7, %xmm1, 14(%rdi)2444; AVX512F-NEXT: LBB14_16: ## %else142445; AVX512F-NEXT: testl $256, %eax ## imm = 0x1002446; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm02447; AVX512F-NEXT: jne LBB14_172448; AVX512F-NEXT: ## %bb.18: ## %else162449; AVX512F-NEXT: testl $512, %eax ## imm = 0x2002450; AVX512F-NEXT: jne LBB14_192451; AVX512F-NEXT: LBB14_20: ## %else182452; AVX512F-NEXT: testl $1024, %eax ## imm = 0x4002453; AVX512F-NEXT: jne LBB14_212454; AVX512F-NEXT: LBB14_22: ## %else202455; AVX512F-NEXT: testl $2048, %eax ## imm = 0x8002456; AVX512F-NEXT: jne LBB14_232457; AVX512F-NEXT: LBB14_24: ## %else222458; AVX512F-NEXT: testl $4096, %eax ## imm = 0x10002459; AVX512F-NEXT: jne LBB14_252460; AVX512F-NEXT: LBB14_26: ## %else242461; AVX512F-NEXT: testl $8192, %eax ## imm = 0x20002462; AVX512F-NEXT: jne LBB14_272463; AVX512F-NEXT: LBB14_28: ## %else262464; AVX512F-NEXT: testl $16384, %eax ## imm = 0x40002465; AVX512F-NEXT: jne LBB14_292466; AVX512F-NEXT: LBB14_30: ## %else282467; AVX512F-NEXT: testl $32768, %eax ## imm = 0x80002468; AVX512F-NEXT: jne LBB14_312469; AVX512F-NEXT: LBB14_32: ## %else302470; AVX512F-NEXT: vzeroupper2471; AVX512F-NEXT: retq2472; AVX512F-NEXT: LBB14_1: ## %cond.store2473; AVX512F-NEXT: vpextrw $0, %xmm1, (%rdi)2474; AVX512F-NEXT: testb $2, %al2475; AVX512F-NEXT: je LBB14_42476; AVX512F-NEXT: LBB14_3: ## %cond.store12477; AVX512F-NEXT: vpextrw $1, %xmm1, 2(%rdi)2478; AVX512F-NEXT: testb $4, %al2479; AVX512F-NEXT: je LBB14_62480; AVX512F-NEXT: LBB14_5: ## %cond.store32481; AVX512F-NEXT: vpextrw $2, %xmm1, 4(%rdi)2482; AVX512F-NEXT: testb $8, %al2483; AVX512F-NEXT: je LBB14_82484; AVX512F-NEXT: LBB14_7: ## %cond.store52485; AVX512F-NEXT: vpextrw $3, %xmm1, 6(%rdi)2486; AVX512F-NEXT: testb $16, %al2487; AVX512F-NEXT: je LBB14_102488; AVX512F-NEXT: LBB14_9: ## %cond.store72489; AVX512F-NEXT: vpextrw $4, %xmm1, 8(%rdi)2490; AVX512F-NEXT: testb $32, %al2491; AVX512F-NEXT: je LBB14_122492; AVX512F-NEXT: LBB14_11: ## %cond.store92493; AVX512F-NEXT: vpextrw $5, %xmm1, 10(%rdi)2494; AVX512F-NEXT: testb $64, %al2495; AVX512F-NEXT: je LBB14_142496; AVX512F-NEXT: LBB14_13: ## %cond.store112497; AVX512F-NEXT: vpextrw $6, %xmm1, 12(%rdi)2498; AVX512F-NEXT: testb %al, %al2499; AVX512F-NEXT: js LBB14_152500; AVX512F-NEXT: jmp LBB14_162501; AVX512F-NEXT: LBB14_17: ## %cond.store152502; AVX512F-NEXT: vpextrw $0, %xmm0, 16(%rdi)2503; AVX512F-NEXT: testl $512, %eax ## imm = 0x2002504; AVX512F-NEXT: je LBB14_202505; AVX512F-NEXT: LBB14_19: ## %cond.store172506; AVX512F-NEXT: vpextrw $1, %xmm0, 18(%rdi)2507; AVX512F-NEXT: testl $1024, %eax ## imm = 0x4002508; AVX512F-NEXT: je LBB14_222509; AVX512F-NEXT: LBB14_21: ## %cond.store192510; AVX512F-NEXT: vpextrw $2, %xmm0, 20(%rdi)2511; AVX512F-NEXT: testl $2048, %eax ## imm = 0x8002512; AVX512F-NEXT: je LBB14_242513; AVX512F-NEXT: LBB14_23: ## %cond.store212514; AVX512F-NEXT: vpextrw $3, %xmm0, 22(%rdi)2515; AVX512F-NEXT: testl $4096, %eax ## imm = 0x10002516; AVX512F-NEXT: je LBB14_262517; AVX512F-NEXT: LBB14_25: ## %cond.store232518; AVX512F-NEXT: vpextrw $4, %xmm0, 24(%rdi)2519; AVX512F-NEXT: testl $8192, %eax ## imm = 0x20002520; AVX512F-NEXT: je LBB14_282521; AVX512F-NEXT: LBB14_27: ## %cond.store252522; AVX512F-NEXT: vpextrw $5, %xmm0, 26(%rdi)2523; AVX512F-NEXT: testl $16384, %eax ## imm = 0x40002524; AVX512F-NEXT: je LBB14_302525; AVX512F-NEXT: LBB14_29: ## %cond.store272526; AVX512F-NEXT: vpextrw $6, %xmm0, 28(%rdi)2527; AVX512F-NEXT: testl $32768, %eax ## imm = 0x80002528; AVX512F-NEXT: je LBB14_322529; AVX512F-NEXT: LBB14_31: ## %cond.store292530; AVX512F-NEXT: vpextrw $7, %xmm0, 30(%rdi)2531; AVX512F-NEXT: vzeroupper2532; AVX512F-NEXT: retq2533;2534; AVX512VLDQ-LABEL: store_v16i16_v16i16:2535; AVX512VLDQ: ## %bb.0:2536; AVX512VLDQ-NEXT: vpxor %xmm2, %xmm2, %xmm22537; AVX512VLDQ-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm02538; AVX512VLDQ-NEXT: vpmovsxwd %ymm0, %zmm02539; AVX512VLDQ-NEXT: vpmovd2m %zmm0, %k02540; AVX512VLDQ-NEXT: kmovw %k0, %eax2541; AVX512VLDQ-NEXT: testb $1, %al2542; AVX512VLDQ-NEXT: jne LBB14_12543; AVX512VLDQ-NEXT: ## %bb.2: ## %else2544; AVX512VLDQ-NEXT: testb $2, %al2545; AVX512VLDQ-NEXT: jne LBB14_32546; AVX512VLDQ-NEXT: LBB14_4: ## %else22547; AVX512VLDQ-NEXT: testb $4, %al2548; AVX512VLDQ-NEXT: jne LBB14_52549; AVX512VLDQ-NEXT: LBB14_6: ## %else42550; AVX512VLDQ-NEXT: testb $8, %al2551; AVX512VLDQ-NEXT: jne LBB14_72552; AVX512VLDQ-NEXT: LBB14_8: ## %else62553; AVX512VLDQ-NEXT: testb $16, %al2554; AVX512VLDQ-NEXT: jne LBB14_92555; AVX512VLDQ-NEXT: LBB14_10: ## %else82556; AVX512VLDQ-NEXT: testb $32, %al2557; AVX512VLDQ-NEXT: jne LBB14_112558; AVX512VLDQ-NEXT: LBB14_12: ## %else102559; AVX512VLDQ-NEXT: testb $64, %al2560; AVX512VLDQ-NEXT: jne LBB14_132561; AVX512VLDQ-NEXT: LBB14_14: ## %else122562; AVX512VLDQ-NEXT: testb %al, %al2563; AVX512VLDQ-NEXT: jns LBB14_162564; AVX512VLDQ-NEXT: LBB14_15: ## %cond.store132565; AVX512VLDQ-NEXT: vpextrw $7, %xmm1, 14(%rdi)2566; AVX512VLDQ-NEXT: LBB14_16: ## %else142567; AVX512VLDQ-NEXT: testl $256, %eax ## imm = 0x1002568; AVX512VLDQ-NEXT: vextracti128 $1, %ymm1, %xmm02569; AVX512VLDQ-NEXT: jne LBB14_172570; AVX512VLDQ-NEXT: ## %bb.18: ## %else162571; AVX512VLDQ-NEXT: testl $512, %eax ## imm = 0x2002572; AVX512VLDQ-NEXT: jne LBB14_192573; AVX512VLDQ-NEXT: LBB14_20: ## %else182574; AVX512VLDQ-NEXT: testl $1024, %eax ## imm = 0x4002575; AVX512VLDQ-NEXT: jne LBB14_212576; AVX512VLDQ-NEXT: LBB14_22: ## %else202577; AVX512VLDQ-NEXT: testl $2048, %eax ## imm = 0x8002578; AVX512VLDQ-NEXT: jne LBB14_232579; AVX512VLDQ-NEXT: LBB14_24: ## %else222580; AVX512VLDQ-NEXT: testl $4096, %eax ## imm = 0x10002581; AVX512VLDQ-NEXT: jne LBB14_252582; AVX512VLDQ-NEXT: LBB14_26: ## %else242583; AVX512VLDQ-NEXT: testl $8192, %eax ## imm = 0x20002584; AVX512VLDQ-NEXT: jne LBB14_272585; AVX512VLDQ-NEXT: LBB14_28: ## %else262586; AVX512VLDQ-NEXT: testl $16384, %eax ## imm = 0x40002587; AVX512VLDQ-NEXT: jne LBB14_292588; AVX512VLDQ-NEXT: LBB14_30: ## %else282589; AVX512VLDQ-NEXT: testl $32768, %eax ## imm = 0x80002590; AVX512VLDQ-NEXT: jne LBB14_312591; AVX512VLDQ-NEXT: LBB14_32: ## %else302592; AVX512VLDQ-NEXT: vzeroupper2593; AVX512VLDQ-NEXT: retq2594; AVX512VLDQ-NEXT: LBB14_1: ## %cond.store2595; AVX512VLDQ-NEXT: vpextrw $0, %xmm1, (%rdi)2596; AVX512VLDQ-NEXT: testb $2, %al2597; AVX512VLDQ-NEXT: je LBB14_42598; AVX512VLDQ-NEXT: LBB14_3: ## %cond.store12599; AVX512VLDQ-NEXT: vpextrw $1, %xmm1, 2(%rdi)2600; AVX512VLDQ-NEXT: testb $4, %al2601; AVX512VLDQ-NEXT: je LBB14_62602; AVX512VLDQ-NEXT: LBB14_5: ## %cond.store32603; AVX512VLDQ-NEXT: vpextrw $2, %xmm1, 4(%rdi)2604; AVX512VLDQ-NEXT: testb $8, %al2605; AVX512VLDQ-NEXT: je LBB14_82606; AVX512VLDQ-NEXT: LBB14_7: ## %cond.store52607; AVX512VLDQ-NEXT: vpextrw $3, %xmm1, 6(%rdi)2608; AVX512VLDQ-NEXT: testb $16, %al2609; AVX512VLDQ-NEXT: je LBB14_102610; AVX512VLDQ-NEXT: LBB14_9: ## %cond.store72611; AVX512VLDQ-NEXT: vpextrw $4, %xmm1, 8(%rdi)2612; AVX512VLDQ-NEXT: testb $32, %al2613; AVX512VLDQ-NEXT: je LBB14_122614; AVX512VLDQ-NEXT: LBB14_11: ## %cond.store92615; AVX512VLDQ-NEXT: vpextrw $5, %xmm1, 10(%rdi)2616; AVX512VLDQ-NEXT: testb $64, %al2617; AVX512VLDQ-NEXT: je LBB14_142618; AVX512VLDQ-NEXT: LBB14_13: ## %cond.store112619; AVX512VLDQ-NEXT: vpextrw $6, %xmm1, 12(%rdi)2620; AVX512VLDQ-NEXT: testb %al, %al2621; AVX512VLDQ-NEXT: js LBB14_152622; AVX512VLDQ-NEXT: jmp LBB14_162623; AVX512VLDQ-NEXT: LBB14_17: ## %cond.store152624; AVX512VLDQ-NEXT: vpextrw $0, %xmm0, 16(%rdi)2625; AVX512VLDQ-NEXT: testl $512, %eax ## imm = 0x2002626; AVX512VLDQ-NEXT: je LBB14_202627; AVX512VLDQ-NEXT: LBB14_19: ## %cond.store172628; AVX512VLDQ-NEXT: vpextrw $1, %xmm0, 18(%rdi)2629; AVX512VLDQ-NEXT: testl $1024, %eax ## imm = 0x4002630; AVX512VLDQ-NEXT: je LBB14_222631; AVX512VLDQ-NEXT: LBB14_21: ## %cond.store192632; AVX512VLDQ-NEXT: vpextrw $2, %xmm0, 20(%rdi)2633; AVX512VLDQ-NEXT: testl $2048, %eax ## imm = 0x8002634; AVX512VLDQ-NEXT: je LBB14_242635; AVX512VLDQ-NEXT: LBB14_23: ## %cond.store212636; AVX512VLDQ-NEXT: vpextrw $3, %xmm0, 22(%rdi)2637; AVX512VLDQ-NEXT: testl $4096, %eax ## imm = 0x10002638; AVX512VLDQ-NEXT: je LBB14_262639; AVX512VLDQ-NEXT: LBB14_25: ## %cond.store232640; AVX512VLDQ-NEXT: vpextrw $4, %xmm0, 24(%rdi)2641; AVX512VLDQ-NEXT: testl $8192, %eax ## imm = 0x20002642; AVX512VLDQ-NEXT: je LBB14_282643; AVX512VLDQ-NEXT: LBB14_27: ## %cond.store252644; AVX512VLDQ-NEXT: vpextrw $5, %xmm0, 26(%rdi)2645; AVX512VLDQ-NEXT: testl $16384, %eax ## imm = 0x40002646; AVX512VLDQ-NEXT: je LBB14_302647; AVX512VLDQ-NEXT: LBB14_29: ## %cond.store272648; AVX512VLDQ-NEXT: vpextrw $6, %xmm0, 28(%rdi)2649; AVX512VLDQ-NEXT: testl $32768, %eax ## imm = 0x80002650; AVX512VLDQ-NEXT: je LBB14_322651; AVX512VLDQ-NEXT: LBB14_31: ## %cond.store292652; AVX512VLDQ-NEXT: vpextrw $7, %xmm0, 30(%rdi)2653; AVX512VLDQ-NEXT: vzeroupper2654; AVX512VLDQ-NEXT: retq2655;2656; AVX512VLBW-LABEL: store_v16i16_v16i16:2657; AVX512VLBW: ## %bb.0:2658; AVX512VLBW-NEXT: vptestnmw %ymm0, %ymm0, %k12659; AVX512VLBW-NEXT: vmovdqu16 %ymm1, (%rdi) {%k1}2660; AVX512VLBW-NEXT: vzeroupper2661; AVX512VLBW-NEXT: retq2662;2663; X86-AVX512-LABEL: store_v16i16_v16i16:2664; X86-AVX512: ## %bb.0:2665; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax2666; X86-AVX512-NEXT: vptestnmw %ymm0, %ymm0, %k12667; X86-AVX512-NEXT: vmovdqu16 %ymm1, (%eax) {%k1}2668; X86-AVX512-NEXT: vzeroupper2669; X86-AVX512-NEXT: retl2670 %mask = icmp eq <16 x i16> %trigger, zeroinitializer2671 call void @llvm.masked.store.v16i16.p0(<16 x i16> %val, ptr %addr, i32 4, <16 x i1> %mask)2672 ret void2673}2674 2675;2676; vXi82677;2678 2679define void @store_v16i8_v16i8(<16 x i8> %trigger, ptr %addr, <16 x i8> %val) nounwind {2680; SSE2-LABEL: store_v16i8_v16i8:2681; SSE2: ## %bb.0:2682; SSE2-NEXT: pxor %xmm2, %xmm22683; SSE2-NEXT: pcmpeqb %xmm0, %xmm22684; SSE2-NEXT: pmovmskb %xmm2, %eax2685; SSE2-NEXT: testb $1, %al2686; SSE2-NEXT: movd %xmm1, %ecx2687; SSE2-NEXT: jne LBB15_12688; SSE2-NEXT: ## %bb.2: ## %else2689; SSE2-NEXT: testb $2, %al2690; SSE2-NEXT: jne LBB15_32691; SSE2-NEXT: LBB15_4: ## %else22692; SSE2-NEXT: testb $4, %al2693; SSE2-NEXT: jne LBB15_52694; SSE2-NEXT: LBB15_6: ## %else42695; SSE2-NEXT: testb $8, %al2696; SSE2-NEXT: je LBB15_82697; SSE2-NEXT: LBB15_7: ## %cond.store52698; SSE2-NEXT: shrl $24, %ecx2699; SSE2-NEXT: movb %cl, 3(%rdi)2700; SSE2-NEXT: LBB15_8: ## %else62701; SSE2-NEXT: testb $16, %al2702; SSE2-NEXT: pextrw $2, %xmm1, %ecx2703; SSE2-NEXT: je LBB15_102704; SSE2-NEXT: ## %bb.9: ## %cond.store72705; SSE2-NEXT: movb %cl, 4(%rdi)2706; SSE2-NEXT: LBB15_10: ## %else82707; SSE2-NEXT: testb $32, %al2708; SSE2-NEXT: je LBB15_122709; SSE2-NEXT: ## %bb.11: ## %cond.store92710; SSE2-NEXT: movb %ch, 5(%rdi)2711; SSE2-NEXT: LBB15_12: ## %else102712; SSE2-NEXT: testb $64, %al2713; SSE2-NEXT: pextrw $3, %xmm1, %ecx2714; SSE2-NEXT: je LBB15_142715; SSE2-NEXT: ## %bb.13: ## %cond.store112716; SSE2-NEXT: movb %cl, 6(%rdi)2717; SSE2-NEXT: LBB15_14: ## %else122718; SSE2-NEXT: testb %al, %al2719; SSE2-NEXT: jns LBB15_162720; SSE2-NEXT: ## %bb.15: ## %cond.store132721; SSE2-NEXT: movb %ch, 7(%rdi)2722; SSE2-NEXT: LBB15_16: ## %else142723; SSE2-NEXT: testl $256, %eax ## imm = 0x1002724; SSE2-NEXT: pextrw $4, %xmm1, %ecx2725; SSE2-NEXT: je LBB15_182726; SSE2-NEXT: ## %bb.17: ## %cond.store152727; SSE2-NEXT: movb %cl, 8(%rdi)2728; SSE2-NEXT: LBB15_18: ## %else162729; SSE2-NEXT: testl $512, %eax ## imm = 0x2002730; SSE2-NEXT: je LBB15_202731; SSE2-NEXT: ## %bb.19: ## %cond.store172732; SSE2-NEXT: movb %ch, 9(%rdi)2733; SSE2-NEXT: LBB15_20: ## %else182734; SSE2-NEXT: testl $1024, %eax ## imm = 0x4002735; SSE2-NEXT: pextrw $5, %xmm1, %ecx2736; SSE2-NEXT: je LBB15_222737; SSE2-NEXT: ## %bb.21: ## %cond.store192738; SSE2-NEXT: movb %cl, 10(%rdi)2739; SSE2-NEXT: LBB15_22: ## %else202740; SSE2-NEXT: testl $2048, %eax ## imm = 0x8002741; SSE2-NEXT: je LBB15_242742; SSE2-NEXT: ## %bb.23: ## %cond.store212743; SSE2-NEXT: movb %ch, 11(%rdi)2744; SSE2-NEXT: LBB15_24: ## %else222745; SSE2-NEXT: testl $4096, %eax ## imm = 0x10002746; SSE2-NEXT: pextrw $6, %xmm1, %ecx2747; SSE2-NEXT: je LBB15_262748; SSE2-NEXT: ## %bb.25: ## %cond.store232749; SSE2-NEXT: movb %cl, 12(%rdi)2750; SSE2-NEXT: LBB15_26: ## %else242751; SSE2-NEXT: testl $8192, %eax ## imm = 0x20002752; SSE2-NEXT: je LBB15_282753; SSE2-NEXT: ## %bb.27: ## %cond.store252754; SSE2-NEXT: movb %ch, 13(%rdi)2755; SSE2-NEXT: LBB15_28: ## %else262756; SSE2-NEXT: testl $16384, %eax ## imm = 0x40002757; SSE2-NEXT: pextrw $7, %xmm1, %ecx2758; SSE2-NEXT: jne LBB15_292759; SSE2-NEXT: ## %bb.30: ## %else282760; SSE2-NEXT: testl $32768, %eax ## imm = 0x80002761; SSE2-NEXT: jne LBB15_312762; SSE2-NEXT: LBB15_32: ## %else302763; SSE2-NEXT: retq2764; SSE2-NEXT: LBB15_1: ## %cond.store2765; SSE2-NEXT: movb %cl, (%rdi)2766; SSE2-NEXT: testb $2, %al2767; SSE2-NEXT: je LBB15_42768; SSE2-NEXT: LBB15_3: ## %cond.store12769; SSE2-NEXT: movb %ch, 1(%rdi)2770; SSE2-NEXT: testb $4, %al2771; SSE2-NEXT: je LBB15_62772; SSE2-NEXT: LBB15_5: ## %cond.store32773; SSE2-NEXT: movl %ecx, %edx2774; SSE2-NEXT: shrl $16, %edx2775; SSE2-NEXT: movb %dl, 2(%rdi)2776; SSE2-NEXT: testb $8, %al2777; SSE2-NEXT: jne LBB15_72778; SSE2-NEXT: jmp LBB15_82779; SSE2-NEXT: LBB15_29: ## %cond.store272780; SSE2-NEXT: movb %cl, 14(%rdi)2781; SSE2-NEXT: testl $32768, %eax ## imm = 0x80002782; SSE2-NEXT: je LBB15_322783; SSE2-NEXT: LBB15_31: ## %cond.store292784; SSE2-NEXT: movb %ch, 15(%rdi)2785; SSE2-NEXT: retq2786;2787; SSE4-LABEL: store_v16i8_v16i8:2788; SSE4: ## %bb.0:2789; SSE4-NEXT: pxor %xmm2, %xmm22790; SSE4-NEXT: pcmpeqb %xmm0, %xmm22791; SSE4-NEXT: pmovmskb %xmm2, %eax2792; SSE4-NEXT: testb $1, %al2793; SSE4-NEXT: jne LBB15_12794; SSE4-NEXT: ## %bb.2: ## %else2795; SSE4-NEXT: testb $2, %al2796; SSE4-NEXT: jne LBB15_32797; SSE4-NEXT: LBB15_4: ## %else22798; SSE4-NEXT: testb $4, %al2799; SSE4-NEXT: jne LBB15_52800; SSE4-NEXT: LBB15_6: ## %else42801; SSE4-NEXT: testb $8, %al2802; SSE4-NEXT: jne LBB15_72803; SSE4-NEXT: LBB15_8: ## %else62804; SSE4-NEXT: testb $16, %al2805; SSE4-NEXT: jne LBB15_92806; SSE4-NEXT: LBB15_10: ## %else82807; SSE4-NEXT: testb $32, %al2808; SSE4-NEXT: jne LBB15_112809; SSE4-NEXT: LBB15_12: ## %else102810; SSE4-NEXT: testb $64, %al2811; SSE4-NEXT: jne LBB15_132812; SSE4-NEXT: LBB15_14: ## %else122813; SSE4-NEXT: testb %al, %al2814; SSE4-NEXT: js LBB15_152815; SSE4-NEXT: LBB15_16: ## %else142816; SSE4-NEXT: testl $256, %eax ## imm = 0x1002817; SSE4-NEXT: jne LBB15_172818; SSE4-NEXT: LBB15_18: ## %else162819; SSE4-NEXT: testl $512, %eax ## imm = 0x2002820; SSE4-NEXT: jne LBB15_192821; SSE4-NEXT: LBB15_20: ## %else182822; SSE4-NEXT: testl $1024, %eax ## imm = 0x4002823; SSE4-NEXT: jne LBB15_212824; SSE4-NEXT: LBB15_22: ## %else202825; SSE4-NEXT: testl $2048, %eax ## imm = 0x8002826; SSE4-NEXT: jne LBB15_232827; SSE4-NEXT: LBB15_24: ## %else222828; SSE4-NEXT: testl $4096, %eax ## imm = 0x10002829; SSE4-NEXT: jne LBB15_252830; SSE4-NEXT: LBB15_26: ## %else242831; SSE4-NEXT: testl $8192, %eax ## imm = 0x20002832; SSE4-NEXT: jne LBB15_272833; SSE4-NEXT: LBB15_28: ## %else262834; SSE4-NEXT: testl $16384, %eax ## imm = 0x40002835; SSE4-NEXT: jne LBB15_292836; SSE4-NEXT: LBB15_30: ## %else282837; SSE4-NEXT: testl $32768, %eax ## imm = 0x80002838; SSE4-NEXT: jne LBB15_312839; SSE4-NEXT: LBB15_32: ## %else302840; SSE4-NEXT: retq2841; SSE4-NEXT: LBB15_1: ## %cond.store2842; SSE4-NEXT: pextrb $0, %xmm1, (%rdi)2843; SSE4-NEXT: testb $2, %al2844; SSE4-NEXT: je LBB15_42845; SSE4-NEXT: LBB15_3: ## %cond.store12846; SSE4-NEXT: pextrb $1, %xmm1, 1(%rdi)2847; SSE4-NEXT: testb $4, %al2848; SSE4-NEXT: je LBB15_62849; SSE4-NEXT: LBB15_5: ## %cond.store32850; SSE4-NEXT: pextrb $2, %xmm1, 2(%rdi)2851; SSE4-NEXT: testb $8, %al2852; SSE4-NEXT: je LBB15_82853; SSE4-NEXT: LBB15_7: ## %cond.store52854; SSE4-NEXT: pextrb $3, %xmm1, 3(%rdi)2855; SSE4-NEXT: testb $16, %al2856; SSE4-NEXT: je LBB15_102857; SSE4-NEXT: LBB15_9: ## %cond.store72858; SSE4-NEXT: pextrb $4, %xmm1, 4(%rdi)2859; SSE4-NEXT: testb $32, %al2860; SSE4-NEXT: je LBB15_122861; SSE4-NEXT: LBB15_11: ## %cond.store92862; SSE4-NEXT: pextrb $5, %xmm1, 5(%rdi)2863; SSE4-NEXT: testb $64, %al2864; SSE4-NEXT: je LBB15_142865; SSE4-NEXT: LBB15_13: ## %cond.store112866; SSE4-NEXT: pextrb $6, %xmm1, 6(%rdi)2867; SSE4-NEXT: testb %al, %al2868; SSE4-NEXT: jns LBB15_162869; SSE4-NEXT: LBB15_15: ## %cond.store132870; SSE4-NEXT: pextrb $7, %xmm1, 7(%rdi)2871; SSE4-NEXT: testl $256, %eax ## imm = 0x1002872; SSE4-NEXT: je LBB15_182873; SSE4-NEXT: LBB15_17: ## %cond.store152874; SSE4-NEXT: pextrb $8, %xmm1, 8(%rdi)2875; SSE4-NEXT: testl $512, %eax ## imm = 0x2002876; SSE4-NEXT: je LBB15_202877; SSE4-NEXT: LBB15_19: ## %cond.store172878; SSE4-NEXT: pextrb $9, %xmm1, 9(%rdi)2879; SSE4-NEXT: testl $1024, %eax ## imm = 0x4002880; SSE4-NEXT: je LBB15_222881; SSE4-NEXT: LBB15_21: ## %cond.store192882; SSE4-NEXT: pextrb $10, %xmm1, 10(%rdi)2883; SSE4-NEXT: testl $2048, %eax ## imm = 0x8002884; SSE4-NEXT: je LBB15_242885; SSE4-NEXT: LBB15_23: ## %cond.store212886; SSE4-NEXT: pextrb $11, %xmm1, 11(%rdi)2887; SSE4-NEXT: testl $4096, %eax ## imm = 0x10002888; SSE4-NEXT: je LBB15_262889; SSE4-NEXT: LBB15_25: ## %cond.store232890; SSE4-NEXT: pextrb $12, %xmm1, 12(%rdi)2891; SSE4-NEXT: testl $8192, %eax ## imm = 0x20002892; SSE4-NEXT: je LBB15_282893; SSE4-NEXT: LBB15_27: ## %cond.store252894; SSE4-NEXT: pextrb $13, %xmm1, 13(%rdi)2895; SSE4-NEXT: testl $16384, %eax ## imm = 0x40002896; SSE4-NEXT: je LBB15_302897; SSE4-NEXT: LBB15_29: ## %cond.store272898; SSE4-NEXT: pextrb $14, %xmm1, 14(%rdi)2899; SSE4-NEXT: testl $32768, %eax ## imm = 0x80002900; SSE4-NEXT: je LBB15_322901; SSE4-NEXT: LBB15_31: ## %cond.store292902; SSE4-NEXT: pextrb $15, %xmm1, 15(%rdi)2903; SSE4-NEXT: retq2904;2905; AVX1OR2-LABEL: store_v16i8_v16i8:2906; AVX1OR2: ## %bb.0:2907; AVX1OR2-NEXT: vpxor %xmm2, %xmm2, %xmm22908; AVX1OR2-NEXT: vpcmpeqb %xmm2, %xmm0, %xmm02909; AVX1OR2-NEXT: vpmovmskb %xmm0, %eax2910; AVX1OR2-NEXT: testb $1, %al2911; AVX1OR2-NEXT: jne LBB15_12912; AVX1OR2-NEXT: ## %bb.2: ## %else2913; AVX1OR2-NEXT: testb $2, %al2914; AVX1OR2-NEXT: jne LBB15_32915; AVX1OR2-NEXT: LBB15_4: ## %else22916; AVX1OR2-NEXT: testb $4, %al2917; AVX1OR2-NEXT: jne LBB15_52918; AVX1OR2-NEXT: LBB15_6: ## %else42919; AVX1OR2-NEXT: testb $8, %al2920; AVX1OR2-NEXT: jne LBB15_72921; AVX1OR2-NEXT: LBB15_8: ## %else62922; AVX1OR2-NEXT: testb $16, %al2923; AVX1OR2-NEXT: jne LBB15_92924; AVX1OR2-NEXT: LBB15_10: ## %else82925; AVX1OR2-NEXT: testb $32, %al2926; AVX1OR2-NEXT: jne LBB15_112927; AVX1OR2-NEXT: LBB15_12: ## %else102928; AVX1OR2-NEXT: testb $64, %al2929; AVX1OR2-NEXT: jne LBB15_132930; AVX1OR2-NEXT: LBB15_14: ## %else122931; AVX1OR2-NEXT: testb %al, %al2932; AVX1OR2-NEXT: js LBB15_152933; AVX1OR2-NEXT: LBB15_16: ## %else142934; AVX1OR2-NEXT: testl $256, %eax ## imm = 0x1002935; AVX1OR2-NEXT: jne LBB15_172936; AVX1OR2-NEXT: LBB15_18: ## %else162937; AVX1OR2-NEXT: testl $512, %eax ## imm = 0x2002938; AVX1OR2-NEXT: jne LBB15_192939; AVX1OR2-NEXT: LBB15_20: ## %else182940; AVX1OR2-NEXT: testl $1024, %eax ## imm = 0x4002941; AVX1OR2-NEXT: jne LBB15_212942; AVX1OR2-NEXT: LBB15_22: ## %else202943; AVX1OR2-NEXT: testl $2048, %eax ## imm = 0x8002944; AVX1OR2-NEXT: jne LBB15_232945; AVX1OR2-NEXT: LBB15_24: ## %else222946; AVX1OR2-NEXT: testl $4096, %eax ## imm = 0x10002947; AVX1OR2-NEXT: jne LBB15_252948; AVX1OR2-NEXT: LBB15_26: ## %else242949; AVX1OR2-NEXT: testl $8192, %eax ## imm = 0x20002950; AVX1OR2-NEXT: jne LBB15_272951; AVX1OR2-NEXT: LBB15_28: ## %else262952; AVX1OR2-NEXT: testl $16384, %eax ## imm = 0x40002953; AVX1OR2-NEXT: jne LBB15_292954; AVX1OR2-NEXT: LBB15_30: ## %else282955; AVX1OR2-NEXT: testl $32768, %eax ## imm = 0x80002956; AVX1OR2-NEXT: jne LBB15_312957; AVX1OR2-NEXT: LBB15_32: ## %else302958; AVX1OR2-NEXT: retq2959; AVX1OR2-NEXT: LBB15_1: ## %cond.store2960; AVX1OR2-NEXT: vpextrb $0, %xmm1, (%rdi)2961; AVX1OR2-NEXT: testb $2, %al2962; AVX1OR2-NEXT: je LBB15_42963; AVX1OR2-NEXT: LBB15_3: ## %cond.store12964; AVX1OR2-NEXT: vpextrb $1, %xmm1, 1(%rdi)2965; AVX1OR2-NEXT: testb $4, %al2966; AVX1OR2-NEXT: je LBB15_62967; AVX1OR2-NEXT: LBB15_5: ## %cond.store32968; AVX1OR2-NEXT: vpextrb $2, %xmm1, 2(%rdi)2969; AVX1OR2-NEXT: testb $8, %al2970; AVX1OR2-NEXT: je LBB15_82971; AVX1OR2-NEXT: LBB15_7: ## %cond.store52972; AVX1OR2-NEXT: vpextrb $3, %xmm1, 3(%rdi)2973; AVX1OR2-NEXT: testb $16, %al2974; AVX1OR2-NEXT: je LBB15_102975; AVX1OR2-NEXT: LBB15_9: ## %cond.store72976; AVX1OR2-NEXT: vpextrb $4, %xmm1, 4(%rdi)2977; AVX1OR2-NEXT: testb $32, %al2978; AVX1OR2-NEXT: je LBB15_122979; AVX1OR2-NEXT: LBB15_11: ## %cond.store92980; AVX1OR2-NEXT: vpextrb $5, %xmm1, 5(%rdi)2981; AVX1OR2-NEXT: testb $64, %al2982; AVX1OR2-NEXT: je LBB15_142983; AVX1OR2-NEXT: LBB15_13: ## %cond.store112984; AVX1OR2-NEXT: vpextrb $6, %xmm1, 6(%rdi)2985; AVX1OR2-NEXT: testb %al, %al2986; AVX1OR2-NEXT: jns LBB15_162987; AVX1OR2-NEXT: LBB15_15: ## %cond.store132988; AVX1OR2-NEXT: vpextrb $7, %xmm1, 7(%rdi)2989; AVX1OR2-NEXT: testl $256, %eax ## imm = 0x1002990; AVX1OR2-NEXT: je LBB15_182991; AVX1OR2-NEXT: LBB15_17: ## %cond.store152992; AVX1OR2-NEXT: vpextrb $8, %xmm1, 8(%rdi)2993; AVX1OR2-NEXT: testl $512, %eax ## imm = 0x2002994; AVX1OR2-NEXT: je LBB15_202995; AVX1OR2-NEXT: LBB15_19: ## %cond.store172996; AVX1OR2-NEXT: vpextrb $9, %xmm1, 9(%rdi)2997; AVX1OR2-NEXT: testl $1024, %eax ## imm = 0x4002998; AVX1OR2-NEXT: je LBB15_222999; AVX1OR2-NEXT: LBB15_21: ## %cond.store193000; AVX1OR2-NEXT: vpextrb $10, %xmm1, 10(%rdi)3001; AVX1OR2-NEXT: testl $2048, %eax ## imm = 0x8003002; AVX1OR2-NEXT: je LBB15_243003; AVX1OR2-NEXT: LBB15_23: ## %cond.store213004; AVX1OR2-NEXT: vpextrb $11, %xmm1, 11(%rdi)3005; AVX1OR2-NEXT: testl $4096, %eax ## imm = 0x10003006; AVX1OR2-NEXT: je LBB15_263007; AVX1OR2-NEXT: LBB15_25: ## %cond.store233008; AVX1OR2-NEXT: vpextrb $12, %xmm1, 12(%rdi)3009; AVX1OR2-NEXT: testl $8192, %eax ## imm = 0x20003010; AVX1OR2-NEXT: je LBB15_283011; AVX1OR2-NEXT: LBB15_27: ## %cond.store253012; AVX1OR2-NEXT: vpextrb $13, %xmm1, 13(%rdi)3013; AVX1OR2-NEXT: testl $16384, %eax ## imm = 0x40003014; AVX1OR2-NEXT: je LBB15_303015; AVX1OR2-NEXT: LBB15_29: ## %cond.store273016; AVX1OR2-NEXT: vpextrb $14, %xmm1, 14(%rdi)3017; AVX1OR2-NEXT: testl $32768, %eax ## imm = 0x80003018; AVX1OR2-NEXT: je LBB15_323019; AVX1OR2-NEXT: LBB15_31: ## %cond.store293020; AVX1OR2-NEXT: vpextrb $15, %xmm1, 15(%rdi)3021; AVX1OR2-NEXT: retq3022;3023; AVX512F-LABEL: store_v16i8_v16i8:3024; AVX512F: ## %bb.0:3025; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm23026; AVX512F-NEXT: vpcmpeqb %xmm2, %xmm0, %xmm03027; AVX512F-NEXT: vpmovmskb %xmm0, %eax3028; AVX512F-NEXT: testb $1, %al3029; AVX512F-NEXT: jne LBB15_13030; AVX512F-NEXT: ## %bb.2: ## %else3031; AVX512F-NEXT: testb $2, %al3032; AVX512F-NEXT: jne LBB15_33033; AVX512F-NEXT: LBB15_4: ## %else23034; AVX512F-NEXT: testb $4, %al3035; AVX512F-NEXT: jne LBB15_53036; AVX512F-NEXT: LBB15_6: ## %else43037; AVX512F-NEXT: testb $8, %al3038; AVX512F-NEXT: jne LBB15_73039; AVX512F-NEXT: LBB15_8: ## %else63040; AVX512F-NEXT: testb $16, %al3041; AVX512F-NEXT: jne LBB15_93042; AVX512F-NEXT: LBB15_10: ## %else83043; AVX512F-NEXT: testb $32, %al3044; AVX512F-NEXT: jne LBB15_113045; AVX512F-NEXT: LBB15_12: ## %else103046; AVX512F-NEXT: testb $64, %al3047; AVX512F-NEXT: jne LBB15_133048; AVX512F-NEXT: LBB15_14: ## %else123049; AVX512F-NEXT: testb %al, %al3050; AVX512F-NEXT: js LBB15_153051; AVX512F-NEXT: LBB15_16: ## %else143052; AVX512F-NEXT: testl $256, %eax ## imm = 0x1003053; AVX512F-NEXT: jne LBB15_173054; AVX512F-NEXT: LBB15_18: ## %else163055; AVX512F-NEXT: testl $512, %eax ## imm = 0x2003056; AVX512F-NEXT: jne LBB15_193057; AVX512F-NEXT: LBB15_20: ## %else183058; AVX512F-NEXT: testl $1024, %eax ## imm = 0x4003059; AVX512F-NEXT: jne LBB15_213060; AVX512F-NEXT: LBB15_22: ## %else203061; AVX512F-NEXT: testl $2048, %eax ## imm = 0x8003062; AVX512F-NEXT: jne LBB15_233063; AVX512F-NEXT: LBB15_24: ## %else223064; AVX512F-NEXT: testl $4096, %eax ## imm = 0x10003065; AVX512F-NEXT: jne LBB15_253066; AVX512F-NEXT: LBB15_26: ## %else243067; AVX512F-NEXT: testl $8192, %eax ## imm = 0x20003068; AVX512F-NEXT: jne LBB15_273069; AVX512F-NEXT: LBB15_28: ## %else263070; AVX512F-NEXT: testl $16384, %eax ## imm = 0x40003071; AVX512F-NEXT: jne LBB15_293072; AVX512F-NEXT: LBB15_30: ## %else283073; AVX512F-NEXT: testl $32768, %eax ## imm = 0x80003074; AVX512F-NEXT: jne LBB15_313075; AVX512F-NEXT: LBB15_32: ## %else303076; AVX512F-NEXT: retq3077; AVX512F-NEXT: LBB15_1: ## %cond.store3078; AVX512F-NEXT: vpextrb $0, %xmm1, (%rdi)3079; AVX512F-NEXT: testb $2, %al3080; AVX512F-NEXT: je LBB15_43081; AVX512F-NEXT: LBB15_3: ## %cond.store13082; AVX512F-NEXT: vpextrb $1, %xmm1, 1(%rdi)3083; AVX512F-NEXT: testb $4, %al3084; AVX512F-NEXT: je LBB15_63085; AVX512F-NEXT: LBB15_5: ## %cond.store33086; AVX512F-NEXT: vpextrb $2, %xmm1, 2(%rdi)3087; AVX512F-NEXT: testb $8, %al3088; AVX512F-NEXT: je LBB15_83089; AVX512F-NEXT: LBB15_7: ## %cond.store53090; AVX512F-NEXT: vpextrb $3, %xmm1, 3(%rdi)3091; AVX512F-NEXT: testb $16, %al3092; AVX512F-NEXT: je LBB15_103093; AVX512F-NEXT: LBB15_9: ## %cond.store73094; AVX512F-NEXT: vpextrb $4, %xmm1, 4(%rdi)3095; AVX512F-NEXT: testb $32, %al3096; AVX512F-NEXT: je LBB15_123097; AVX512F-NEXT: LBB15_11: ## %cond.store93098; AVX512F-NEXT: vpextrb $5, %xmm1, 5(%rdi)3099; AVX512F-NEXT: testb $64, %al3100; AVX512F-NEXT: je LBB15_143101; AVX512F-NEXT: LBB15_13: ## %cond.store113102; AVX512F-NEXT: vpextrb $6, %xmm1, 6(%rdi)3103; AVX512F-NEXT: testb %al, %al3104; AVX512F-NEXT: jns LBB15_163105; AVX512F-NEXT: LBB15_15: ## %cond.store133106; AVX512F-NEXT: vpextrb $7, %xmm1, 7(%rdi)3107; AVX512F-NEXT: testl $256, %eax ## imm = 0x1003108; AVX512F-NEXT: je LBB15_183109; AVX512F-NEXT: LBB15_17: ## %cond.store153110; AVX512F-NEXT: vpextrb $8, %xmm1, 8(%rdi)3111; AVX512F-NEXT: testl $512, %eax ## imm = 0x2003112; AVX512F-NEXT: je LBB15_203113; AVX512F-NEXT: LBB15_19: ## %cond.store173114; AVX512F-NEXT: vpextrb $9, %xmm1, 9(%rdi)3115; AVX512F-NEXT: testl $1024, %eax ## imm = 0x4003116; AVX512F-NEXT: je LBB15_223117; AVX512F-NEXT: LBB15_21: ## %cond.store193118; AVX512F-NEXT: vpextrb $10, %xmm1, 10(%rdi)3119; AVX512F-NEXT: testl $2048, %eax ## imm = 0x8003120; AVX512F-NEXT: je LBB15_243121; AVX512F-NEXT: LBB15_23: ## %cond.store213122; AVX512F-NEXT: vpextrb $11, %xmm1, 11(%rdi)3123; AVX512F-NEXT: testl $4096, %eax ## imm = 0x10003124; AVX512F-NEXT: je LBB15_263125; AVX512F-NEXT: LBB15_25: ## %cond.store233126; AVX512F-NEXT: vpextrb $12, %xmm1, 12(%rdi)3127; AVX512F-NEXT: testl $8192, %eax ## imm = 0x20003128; AVX512F-NEXT: je LBB15_283129; AVX512F-NEXT: LBB15_27: ## %cond.store253130; AVX512F-NEXT: vpextrb $13, %xmm1, 13(%rdi)3131; AVX512F-NEXT: testl $16384, %eax ## imm = 0x40003132; AVX512F-NEXT: je LBB15_303133; AVX512F-NEXT: LBB15_29: ## %cond.store273134; AVX512F-NEXT: vpextrb $14, %xmm1, 14(%rdi)3135; AVX512F-NEXT: testl $32768, %eax ## imm = 0x80003136; AVX512F-NEXT: je LBB15_323137; AVX512F-NEXT: LBB15_31: ## %cond.store293138; AVX512F-NEXT: vpextrb $15, %xmm1, 15(%rdi)3139; AVX512F-NEXT: retq3140;3141; AVX512VLDQ-LABEL: store_v16i8_v16i8:3142; AVX512VLDQ: ## %bb.0:3143; AVX512VLDQ-NEXT: vpxor %xmm2, %xmm2, %xmm23144; AVX512VLDQ-NEXT: vpcmpeqb %xmm2, %xmm0, %xmm03145; AVX512VLDQ-NEXT: vpmovmskb %xmm0, %eax3146; AVX512VLDQ-NEXT: testb $1, %al3147; AVX512VLDQ-NEXT: jne LBB15_13148; AVX512VLDQ-NEXT: ## %bb.2: ## %else3149; AVX512VLDQ-NEXT: testb $2, %al3150; AVX512VLDQ-NEXT: jne LBB15_33151; AVX512VLDQ-NEXT: LBB15_4: ## %else23152; AVX512VLDQ-NEXT: testb $4, %al3153; AVX512VLDQ-NEXT: jne LBB15_53154; AVX512VLDQ-NEXT: LBB15_6: ## %else43155; AVX512VLDQ-NEXT: testb $8, %al3156; AVX512VLDQ-NEXT: jne LBB15_73157; AVX512VLDQ-NEXT: LBB15_8: ## %else63158; AVX512VLDQ-NEXT: testb $16, %al3159; AVX512VLDQ-NEXT: jne LBB15_93160; AVX512VLDQ-NEXT: LBB15_10: ## %else83161; AVX512VLDQ-NEXT: testb $32, %al3162; AVX512VLDQ-NEXT: jne LBB15_113163; AVX512VLDQ-NEXT: LBB15_12: ## %else103164; AVX512VLDQ-NEXT: testb $64, %al3165; AVX512VLDQ-NEXT: jne LBB15_133166; AVX512VLDQ-NEXT: LBB15_14: ## %else123167; AVX512VLDQ-NEXT: testb %al, %al3168; AVX512VLDQ-NEXT: js LBB15_153169; AVX512VLDQ-NEXT: LBB15_16: ## %else143170; AVX512VLDQ-NEXT: testl $256, %eax ## imm = 0x1003171; AVX512VLDQ-NEXT: jne LBB15_173172; AVX512VLDQ-NEXT: LBB15_18: ## %else163173; AVX512VLDQ-NEXT: testl $512, %eax ## imm = 0x2003174; AVX512VLDQ-NEXT: jne LBB15_193175; AVX512VLDQ-NEXT: LBB15_20: ## %else183176; AVX512VLDQ-NEXT: testl $1024, %eax ## imm = 0x4003177; AVX512VLDQ-NEXT: jne LBB15_213178; AVX512VLDQ-NEXT: LBB15_22: ## %else203179; AVX512VLDQ-NEXT: testl $2048, %eax ## imm = 0x8003180; AVX512VLDQ-NEXT: jne LBB15_233181; AVX512VLDQ-NEXT: LBB15_24: ## %else223182; AVX512VLDQ-NEXT: testl $4096, %eax ## imm = 0x10003183; AVX512VLDQ-NEXT: jne LBB15_253184; AVX512VLDQ-NEXT: LBB15_26: ## %else243185; AVX512VLDQ-NEXT: testl $8192, %eax ## imm = 0x20003186; AVX512VLDQ-NEXT: jne LBB15_273187; AVX512VLDQ-NEXT: LBB15_28: ## %else263188; AVX512VLDQ-NEXT: testl $16384, %eax ## imm = 0x40003189; AVX512VLDQ-NEXT: jne LBB15_293190; AVX512VLDQ-NEXT: LBB15_30: ## %else283191; AVX512VLDQ-NEXT: testl $32768, %eax ## imm = 0x80003192; AVX512VLDQ-NEXT: jne LBB15_313193; AVX512VLDQ-NEXT: LBB15_32: ## %else303194; AVX512VLDQ-NEXT: retq3195; AVX512VLDQ-NEXT: LBB15_1: ## %cond.store3196; AVX512VLDQ-NEXT: vpextrb $0, %xmm1, (%rdi)3197; AVX512VLDQ-NEXT: testb $2, %al3198; AVX512VLDQ-NEXT: je LBB15_43199; AVX512VLDQ-NEXT: LBB15_3: ## %cond.store13200; AVX512VLDQ-NEXT: vpextrb $1, %xmm1, 1(%rdi)3201; AVX512VLDQ-NEXT: testb $4, %al3202; AVX512VLDQ-NEXT: je LBB15_63203; AVX512VLDQ-NEXT: LBB15_5: ## %cond.store33204; AVX512VLDQ-NEXT: vpextrb $2, %xmm1, 2(%rdi)3205; AVX512VLDQ-NEXT: testb $8, %al3206; AVX512VLDQ-NEXT: je LBB15_83207; AVX512VLDQ-NEXT: LBB15_7: ## %cond.store53208; AVX512VLDQ-NEXT: vpextrb $3, %xmm1, 3(%rdi)3209; AVX512VLDQ-NEXT: testb $16, %al3210; AVX512VLDQ-NEXT: je LBB15_103211; AVX512VLDQ-NEXT: LBB15_9: ## %cond.store73212; AVX512VLDQ-NEXT: vpextrb $4, %xmm1, 4(%rdi)3213; AVX512VLDQ-NEXT: testb $32, %al3214; AVX512VLDQ-NEXT: je LBB15_123215; AVX512VLDQ-NEXT: LBB15_11: ## %cond.store93216; AVX512VLDQ-NEXT: vpextrb $5, %xmm1, 5(%rdi)3217; AVX512VLDQ-NEXT: testb $64, %al3218; AVX512VLDQ-NEXT: je LBB15_143219; AVX512VLDQ-NEXT: LBB15_13: ## %cond.store113220; AVX512VLDQ-NEXT: vpextrb $6, %xmm1, 6(%rdi)3221; AVX512VLDQ-NEXT: testb %al, %al3222; AVX512VLDQ-NEXT: jns LBB15_163223; AVX512VLDQ-NEXT: LBB15_15: ## %cond.store133224; AVX512VLDQ-NEXT: vpextrb $7, %xmm1, 7(%rdi)3225; AVX512VLDQ-NEXT: testl $256, %eax ## imm = 0x1003226; AVX512VLDQ-NEXT: je LBB15_183227; AVX512VLDQ-NEXT: LBB15_17: ## %cond.store153228; AVX512VLDQ-NEXT: vpextrb $8, %xmm1, 8(%rdi)3229; AVX512VLDQ-NEXT: testl $512, %eax ## imm = 0x2003230; AVX512VLDQ-NEXT: je LBB15_203231; AVX512VLDQ-NEXT: LBB15_19: ## %cond.store173232; AVX512VLDQ-NEXT: vpextrb $9, %xmm1, 9(%rdi)3233; AVX512VLDQ-NEXT: testl $1024, %eax ## imm = 0x4003234; AVX512VLDQ-NEXT: je LBB15_223235; AVX512VLDQ-NEXT: LBB15_21: ## %cond.store193236; AVX512VLDQ-NEXT: vpextrb $10, %xmm1, 10(%rdi)3237; AVX512VLDQ-NEXT: testl $2048, %eax ## imm = 0x8003238; AVX512VLDQ-NEXT: je LBB15_243239; AVX512VLDQ-NEXT: LBB15_23: ## %cond.store213240; AVX512VLDQ-NEXT: vpextrb $11, %xmm1, 11(%rdi)3241; AVX512VLDQ-NEXT: testl $4096, %eax ## imm = 0x10003242; AVX512VLDQ-NEXT: je LBB15_263243; AVX512VLDQ-NEXT: LBB15_25: ## %cond.store233244; AVX512VLDQ-NEXT: vpextrb $12, %xmm1, 12(%rdi)3245; AVX512VLDQ-NEXT: testl $8192, %eax ## imm = 0x20003246; AVX512VLDQ-NEXT: je LBB15_283247; AVX512VLDQ-NEXT: LBB15_27: ## %cond.store253248; AVX512VLDQ-NEXT: vpextrb $13, %xmm1, 13(%rdi)3249; AVX512VLDQ-NEXT: testl $16384, %eax ## imm = 0x40003250; AVX512VLDQ-NEXT: je LBB15_303251; AVX512VLDQ-NEXT: LBB15_29: ## %cond.store273252; AVX512VLDQ-NEXT: vpextrb $14, %xmm1, 14(%rdi)3253; AVX512VLDQ-NEXT: testl $32768, %eax ## imm = 0x80003254; AVX512VLDQ-NEXT: je LBB15_323255; AVX512VLDQ-NEXT: LBB15_31: ## %cond.store293256; AVX512VLDQ-NEXT: vpextrb $15, %xmm1, 15(%rdi)3257; AVX512VLDQ-NEXT: retq3258;3259; AVX512VLBW-LABEL: store_v16i8_v16i8:3260; AVX512VLBW: ## %bb.0:3261; AVX512VLBW-NEXT: vptestnmb %xmm0, %xmm0, %k13262; AVX512VLBW-NEXT: vmovdqu8 %xmm1, (%rdi) {%k1}3263; AVX512VLBW-NEXT: retq3264;3265; X86-AVX512-LABEL: store_v16i8_v16i8:3266; X86-AVX512: ## %bb.0:3267; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax3268; X86-AVX512-NEXT: vptestnmb %xmm0, %xmm0, %k13269; X86-AVX512-NEXT: vmovdqu8 %xmm1, (%eax) {%k1}3270; X86-AVX512-NEXT: retl3271 %mask = icmp eq <16 x i8> %trigger, zeroinitializer3272 call void @llvm.masked.store.v16i8.p0(<16 x i8> %val, ptr %addr, i32 4, <16 x i1> %mask)3273 ret void3274}3275 3276define void @store_v32i8_v32i8(<32 x i8> %trigger, ptr %addr, <32 x i8> %val) nounwind {3277; SSE2-LABEL: store_v32i8_v32i8:3278; SSE2: ## %bb.0:3279; SSE2-NEXT: pxor %xmm4, %xmm43280; SSE2-NEXT: pcmpeqb %xmm4, %xmm03281; SSE2-NEXT: pmovmskb %xmm0, %ecx3282; SSE2-NEXT: pcmpeqb %xmm4, %xmm13283; SSE2-NEXT: pmovmskb %xmm1, %eax3284; SSE2-NEXT: shll $16, %eax3285; SSE2-NEXT: orl %ecx, %eax3286; SSE2-NEXT: testb $1, %al3287; SSE2-NEXT: movd %xmm2, %ecx3288; SSE2-NEXT: jne LBB16_13289; SSE2-NEXT: ## %bb.2: ## %else3290; SSE2-NEXT: testb $2, %al3291; SSE2-NEXT: jne LBB16_33292; SSE2-NEXT: LBB16_4: ## %else23293; SSE2-NEXT: testb $4, %al3294; SSE2-NEXT: jne LBB16_53295; SSE2-NEXT: LBB16_6: ## %else43296; SSE2-NEXT: testb $8, %al3297; SSE2-NEXT: je LBB16_83298; SSE2-NEXT: LBB16_7: ## %cond.store53299; SSE2-NEXT: shrl $24, %ecx3300; SSE2-NEXT: movb %cl, 3(%rdi)3301; SSE2-NEXT: LBB16_8: ## %else63302; SSE2-NEXT: testb $16, %al3303; SSE2-NEXT: pextrw $2, %xmm2, %ecx3304; SSE2-NEXT: je LBB16_103305; SSE2-NEXT: ## %bb.9: ## %cond.store73306; SSE2-NEXT: movb %cl, 4(%rdi)3307; SSE2-NEXT: LBB16_10: ## %else83308; SSE2-NEXT: testb $32, %al3309; SSE2-NEXT: je LBB16_123310; SSE2-NEXT: ## %bb.11: ## %cond.store93311; SSE2-NEXT: movb %ch, 5(%rdi)3312; SSE2-NEXT: LBB16_12: ## %else103313; SSE2-NEXT: testb $64, %al3314; SSE2-NEXT: pextrw $3, %xmm2, %ecx3315; SSE2-NEXT: je LBB16_143316; SSE2-NEXT: ## %bb.13: ## %cond.store113317; SSE2-NEXT: movb %cl, 6(%rdi)3318; SSE2-NEXT: LBB16_14: ## %else123319; SSE2-NEXT: testb %al, %al3320; SSE2-NEXT: jns LBB16_163321; SSE2-NEXT: ## %bb.15: ## %cond.store133322; SSE2-NEXT: movb %ch, 7(%rdi)3323; SSE2-NEXT: LBB16_16: ## %else143324; SSE2-NEXT: testl $256, %eax ## imm = 0x1003325; SSE2-NEXT: pextrw $4, %xmm2, %ecx3326; SSE2-NEXT: je LBB16_183327; SSE2-NEXT: ## %bb.17: ## %cond.store153328; SSE2-NEXT: movb %cl, 8(%rdi)3329; SSE2-NEXT: LBB16_18: ## %else163330; SSE2-NEXT: testl $512, %eax ## imm = 0x2003331; SSE2-NEXT: je LBB16_203332; SSE2-NEXT: ## %bb.19: ## %cond.store173333; SSE2-NEXT: movb %ch, 9(%rdi)3334; SSE2-NEXT: LBB16_20: ## %else183335; SSE2-NEXT: testl $1024, %eax ## imm = 0x4003336; SSE2-NEXT: pextrw $5, %xmm2, %ecx3337; SSE2-NEXT: je LBB16_223338; SSE2-NEXT: ## %bb.21: ## %cond.store193339; SSE2-NEXT: movb %cl, 10(%rdi)3340; SSE2-NEXT: LBB16_22: ## %else203341; SSE2-NEXT: testl $2048, %eax ## imm = 0x8003342; SSE2-NEXT: je LBB16_243343; SSE2-NEXT: ## %bb.23: ## %cond.store213344; SSE2-NEXT: movb %ch, 11(%rdi)3345; SSE2-NEXT: LBB16_24: ## %else223346; SSE2-NEXT: testl $4096, %eax ## imm = 0x10003347; SSE2-NEXT: pextrw $6, %xmm2, %ecx3348; SSE2-NEXT: je LBB16_263349; SSE2-NEXT: ## %bb.25: ## %cond.store233350; SSE2-NEXT: movb %cl, 12(%rdi)3351; SSE2-NEXT: LBB16_26: ## %else243352; SSE2-NEXT: testl $8192, %eax ## imm = 0x20003353; SSE2-NEXT: je LBB16_283354; SSE2-NEXT: ## %bb.27: ## %cond.store253355; SSE2-NEXT: movb %ch, 13(%rdi)3356; SSE2-NEXT: LBB16_28: ## %else263357; SSE2-NEXT: testl $16384, %eax ## imm = 0x40003358; SSE2-NEXT: pextrw $7, %xmm2, %ecx3359; SSE2-NEXT: je LBB16_303360; SSE2-NEXT: ## %bb.29: ## %cond.store273361; SSE2-NEXT: movb %cl, 14(%rdi)3362; SSE2-NEXT: LBB16_30: ## %else283363; SSE2-NEXT: testw %ax, %ax3364; SSE2-NEXT: jns LBB16_323365; SSE2-NEXT: ## %bb.31: ## %cond.store293366; SSE2-NEXT: movb %ch, 15(%rdi)3367; SSE2-NEXT: LBB16_32: ## %else303368; SSE2-NEXT: testl $65536, %eax ## imm = 0x100003369; SSE2-NEXT: movd %xmm3, %ecx3370; SSE2-NEXT: jne LBB16_333371; SSE2-NEXT: ## %bb.34: ## %else323372; SSE2-NEXT: testl $131072, %eax ## imm = 0x200003373; SSE2-NEXT: jne LBB16_353374; SSE2-NEXT: LBB16_36: ## %else343375; SSE2-NEXT: testl $262144, %eax ## imm = 0x400003376; SSE2-NEXT: jne LBB16_373377; SSE2-NEXT: LBB16_38: ## %else363378; SSE2-NEXT: testl $524288, %eax ## imm = 0x800003379; SSE2-NEXT: je LBB16_403380; SSE2-NEXT: LBB16_39: ## %cond.store373381; SSE2-NEXT: shrl $24, %ecx3382; SSE2-NEXT: movb %cl, 19(%rdi)3383; SSE2-NEXT: LBB16_40: ## %else383384; SSE2-NEXT: testl $1048576, %eax ## imm = 0x1000003385; SSE2-NEXT: pextrw $2, %xmm3, %ecx3386; SSE2-NEXT: je LBB16_423387; SSE2-NEXT: ## %bb.41: ## %cond.store393388; SSE2-NEXT: movb %cl, 20(%rdi)3389; SSE2-NEXT: LBB16_42: ## %else403390; SSE2-NEXT: testl $2097152, %eax ## imm = 0x2000003391; SSE2-NEXT: je LBB16_443392; SSE2-NEXT: ## %bb.43: ## %cond.store413393; SSE2-NEXT: movb %ch, 21(%rdi)3394; SSE2-NEXT: LBB16_44: ## %else423395; SSE2-NEXT: testl $4194304, %eax ## imm = 0x4000003396; SSE2-NEXT: pextrw $3, %xmm3, %ecx3397; SSE2-NEXT: je LBB16_463398; SSE2-NEXT: ## %bb.45: ## %cond.store433399; SSE2-NEXT: movb %cl, 22(%rdi)3400; SSE2-NEXT: LBB16_46: ## %else443401; SSE2-NEXT: testl $8388608, %eax ## imm = 0x8000003402; SSE2-NEXT: je LBB16_483403; SSE2-NEXT: ## %bb.47: ## %cond.store453404; SSE2-NEXT: movb %ch, 23(%rdi)3405; SSE2-NEXT: LBB16_48: ## %else463406; SSE2-NEXT: testl $16777216, %eax ## imm = 0x10000003407; SSE2-NEXT: pextrw $4, %xmm3, %ecx3408; SSE2-NEXT: je LBB16_503409; SSE2-NEXT: ## %bb.49: ## %cond.store473410; SSE2-NEXT: movb %cl, 24(%rdi)3411; SSE2-NEXT: LBB16_50: ## %else483412; SSE2-NEXT: testl $33554432, %eax ## imm = 0x20000003413; SSE2-NEXT: je LBB16_523414; SSE2-NEXT: ## %bb.51: ## %cond.store493415; SSE2-NEXT: movb %ch, 25(%rdi)3416; SSE2-NEXT: LBB16_52: ## %else503417; SSE2-NEXT: testl $67108864, %eax ## imm = 0x40000003418; SSE2-NEXT: pextrw $5, %xmm3, %ecx3419; SSE2-NEXT: je LBB16_543420; SSE2-NEXT: ## %bb.53: ## %cond.store513421; SSE2-NEXT: movb %cl, 26(%rdi)3422; SSE2-NEXT: LBB16_54: ## %else523423; SSE2-NEXT: testl $134217728, %eax ## imm = 0x80000003424; SSE2-NEXT: je LBB16_563425; SSE2-NEXT: ## %bb.55: ## %cond.store533426; SSE2-NEXT: movb %ch, 27(%rdi)3427; SSE2-NEXT: LBB16_56: ## %else543428; SSE2-NEXT: testl $268435456, %eax ## imm = 0x100000003429; SSE2-NEXT: pextrw $6, %xmm3, %ecx3430; SSE2-NEXT: je LBB16_583431; SSE2-NEXT: ## %bb.57: ## %cond.store553432; SSE2-NEXT: movb %cl, 28(%rdi)3433; SSE2-NEXT: LBB16_58: ## %else563434; SSE2-NEXT: testl $536870912, %eax ## imm = 0x200000003435; SSE2-NEXT: je LBB16_603436; SSE2-NEXT: ## %bb.59: ## %cond.store573437; SSE2-NEXT: movb %ch, 29(%rdi)3438; SSE2-NEXT: LBB16_60: ## %else583439; SSE2-NEXT: testl $1073741824, %eax ## imm = 0x400000003440; SSE2-NEXT: pextrw $7, %xmm3, %ecx3441; SSE2-NEXT: jne LBB16_613442; SSE2-NEXT: ## %bb.62: ## %else603443; SSE2-NEXT: testl $-2147483648, %eax ## imm = 0x800000003444; SSE2-NEXT: jne LBB16_633445; SSE2-NEXT: LBB16_64: ## %else623446; SSE2-NEXT: retq3447; SSE2-NEXT: LBB16_1: ## %cond.store3448; SSE2-NEXT: movb %cl, (%rdi)3449; SSE2-NEXT: testb $2, %al3450; SSE2-NEXT: je LBB16_43451; SSE2-NEXT: LBB16_3: ## %cond.store13452; SSE2-NEXT: movb %ch, 1(%rdi)3453; SSE2-NEXT: testb $4, %al3454; SSE2-NEXT: je LBB16_63455; SSE2-NEXT: LBB16_5: ## %cond.store33456; SSE2-NEXT: movl %ecx, %edx3457; SSE2-NEXT: shrl $16, %edx3458; SSE2-NEXT: movb %dl, 2(%rdi)3459; SSE2-NEXT: testb $8, %al3460; SSE2-NEXT: jne LBB16_73461; SSE2-NEXT: jmp LBB16_83462; SSE2-NEXT: LBB16_33: ## %cond.store313463; SSE2-NEXT: movb %cl, 16(%rdi)3464; SSE2-NEXT: testl $131072, %eax ## imm = 0x200003465; SSE2-NEXT: je LBB16_363466; SSE2-NEXT: LBB16_35: ## %cond.store333467; SSE2-NEXT: movb %ch, 17(%rdi)3468; SSE2-NEXT: testl $262144, %eax ## imm = 0x400003469; SSE2-NEXT: je LBB16_383470; SSE2-NEXT: LBB16_37: ## %cond.store353471; SSE2-NEXT: movl %ecx, %edx3472; SSE2-NEXT: shrl $16, %edx3473; SSE2-NEXT: movb %dl, 18(%rdi)3474; SSE2-NEXT: testl $524288, %eax ## imm = 0x800003475; SSE2-NEXT: jne LBB16_393476; SSE2-NEXT: jmp LBB16_403477; SSE2-NEXT: LBB16_61: ## %cond.store593478; SSE2-NEXT: movb %cl, 30(%rdi)3479; SSE2-NEXT: testl $-2147483648, %eax ## imm = 0x800000003480; SSE2-NEXT: je LBB16_643481; SSE2-NEXT: LBB16_63: ## %cond.store613482; SSE2-NEXT: movb %ch, 31(%rdi)3483; SSE2-NEXT: retq3484;3485; SSE4-LABEL: store_v32i8_v32i8:3486; SSE4: ## %bb.0:3487; SSE4-NEXT: pxor %xmm4, %xmm43488; SSE4-NEXT: pcmpeqb %xmm4, %xmm03489; SSE4-NEXT: pmovmskb %xmm0, %ecx3490; SSE4-NEXT: pcmpeqb %xmm4, %xmm13491; SSE4-NEXT: pmovmskb %xmm1, %eax3492; SSE4-NEXT: shll $16, %eax3493; SSE4-NEXT: orl %ecx, %eax3494; SSE4-NEXT: testb $1, %al3495; SSE4-NEXT: jne LBB16_13496; SSE4-NEXT: ## %bb.2: ## %else3497; SSE4-NEXT: testb $2, %al3498; SSE4-NEXT: jne LBB16_33499; SSE4-NEXT: LBB16_4: ## %else23500; SSE4-NEXT: testb $4, %al3501; SSE4-NEXT: jne LBB16_53502; SSE4-NEXT: LBB16_6: ## %else43503; SSE4-NEXT: testb $8, %al3504; SSE4-NEXT: jne LBB16_73505; SSE4-NEXT: LBB16_8: ## %else63506; SSE4-NEXT: testb $16, %al3507; SSE4-NEXT: jne LBB16_93508; SSE4-NEXT: LBB16_10: ## %else83509; SSE4-NEXT: testb $32, %al3510; SSE4-NEXT: jne LBB16_113511; SSE4-NEXT: LBB16_12: ## %else103512; SSE4-NEXT: testb $64, %al3513; SSE4-NEXT: jne LBB16_133514; SSE4-NEXT: LBB16_14: ## %else123515; SSE4-NEXT: testb %al, %al3516; SSE4-NEXT: js LBB16_153517; SSE4-NEXT: LBB16_16: ## %else143518; SSE4-NEXT: testl $256, %eax ## imm = 0x1003519; SSE4-NEXT: jne LBB16_173520; SSE4-NEXT: LBB16_18: ## %else163521; SSE4-NEXT: testl $512, %eax ## imm = 0x2003522; SSE4-NEXT: jne LBB16_193523; SSE4-NEXT: LBB16_20: ## %else183524; SSE4-NEXT: testl $1024, %eax ## imm = 0x4003525; SSE4-NEXT: jne LBB16_213526; SSE4-NEXT: LBB16_22: ## %else203527; SSE4-NEXT: testl $2048, %eax ## imm = 0x8003528; SSE4-NEXT: jne LBB16_233529; SSE4-NEXT: LBB16_24: ## %else223530; SSE4-NEXT: testl $4096, %eax ## imm = 0x10003531; SSE4-NEXT: jne LBB16_253532; SSE4-NEXT: LBB16_26: ## %else243533; SSE4-NEXT: testl $8192, %eax ## imm = 0x20003534; SSE4-NEXT: jne LBB16_273535; SSE4-NEXT: LBB16_28: ## %else263536; SSE4-NEXT: testl $16384, %eax ## imm = 0x40003537; SSE4-NEXT: jne LBB16_293538; SSE4-NEXT: LBB16_30: ## %else283539; SSE4-NEXT: testw %ax, %ax3540; SSE4-NEXT: js LBB16_313541; SSE4-NEXT: LBB16_32: ## %else303542; SSE4-NEXT: testl $65536, %eax ## imm = 0x100003543; SSE4-NEXT: jne LBB16_333544; SSE4-NEXT: LBB16_34: ## %else323545; SSE4-NEXT: testl $131072, %eax ## imm = 0x200003546; SSE4-NEXT: jne LBB16_353547; SSE4-NEXT: LBB16_36: ## %else343548; SSE4-NEXT: testl $262144, %eax ## imm = 0x400003549; SSE4-NEXT: jne LBB16_373550; SSE4-NEXT: LBB16_38: ## %else363551; SSE4-NEXT: testl $524288, %eax ## imm = 0x800003552; SSE4-NEXT: jne LBB16_393553; SSE4-NEXT: LBB16_40: ## %else383554; SSE4-NEXT: testl $1048576, %eax ## imm = 0x1000003555; SSE4-NEXT: jne LBB16_413556; SSE4-NEXT: LBB16_42: ## %else403557; SSE4-NEXT: testl $2097152, %eax ## imm = 0x2000003558; SSE4-NEXT: jne LBB16_433559; SSE4-NEXT: LBB16_44: ## %else423560; SSE4-NEXT: testl $4194304, %eax ## imm = 0x4000003561; SSE4-NEXT: jne LBB16_453562; SSE4-NEXT: LBB16_46: ## %else443563; SSE4-NEXT: testl $8388608, %eax ## imm = 0x8000003564; SSE4-NEXT: jne LBB16_473565; SSE4-NEXT: LBB16_48: ## %else463566; SSE4-NEXT: testl $16777216, %eax ## imm = 0x10000003567; SSE4-NEXT: jne LBB16_493568; SSE4-NEXT: LBB16_50: ## %else483569; SSE4-NEXT: testl $33554432, %eax ## imm = 0x20000003570; SSE4-NEXT: jne LBB16_513571; SSE4-NEXT: LBB16_52: ## %else503572; SSE4-NEXT: testl $67108864, %eax ## imm = 0x40000003573; SSE4-NEXT: jne LBB16_533574; SSE4-NEXT: LBB16_54: ## %else523575; SSE4-NEXT: testl $134217728, %eax ## imm = 0x80000003576; SSE4-NEXT: jne LBB16_553577; SSE4-NEXT: LBB16_56: ## %else543578; SSE4-NEXT: testl $268435456, %eax ## imm = 0x100000003579; SSE4-NEXT: jne LBB16_573580; SSE4-NEXT: LBB16_58: ## %else563581; SSE4-NEXT: testl $536870912, %eax ## imm = 0x200000003582; SSE4-NEXT: jne LBB16_593583; SSE4-NEXT: LBB16_60: ## %else583584; SSE4-NEXT: testl $1073741824, %eax ## imm = 0x400000003585; SSE4-NEXT: jne LBB16_613586; SSE4-NEXT: LBB16_62: ## %else603587; SSE4-NEXT: testl $-2147483648, %eax ## imm = 0x800000003588; SSE4-NEXT: jne LBB16_633589; SSE4-NEXT: LBB16_64: ## %else623590; SSE4-NEXT: retq3591; SSE4-NEXT: LBB16_1: ## %cond.store3592; SSE4-NEXT: pextrb $0, %xmm2, (%rdi)3593; SSE4-NEXT: testb $2, %al3594; SSE4-NEXT: je LBB16_43595; SSE4-NEXT: LBB16_3: ## %cond.store13596; SSE4-NEXT: pextrb $1, %xmm2, 1(%rdi)3597; SSE4-NEXT: testb $4, %al3598; SSE4-NEXT: je LBB16_63599; SSE4-NEXT: LBB16_5: ## %cond.store33600; SSE4-NEXT: pextrb $2, %xmm2, 2(%rdi)3601; SSE4-NEXT: testb $8, %al3602; SSE4-NEXT: je LBB16_83603; SSE4-NEXT: LBB16_7: ## %cond.store53604; SSE4-NEXT: pextrb $3, %xmm2, 3(%rdi)3605; SSE4-NEXT: testb $16, %al3606; SSE4-NEXT: je LBB16_103607; SSE4-NEXT: LBB16_9: ## %cond.store73608; SSE4-NEXT: pextrb $4, %xmm2, 4(%rdi)3609; SSE4-NEXT: testb $32, %al3610; SSE4-NEXT: je LBB16_123611; SSE4-NEXT: LBB16_11: ## %cond.store93612; SSE4-NEXT: pextrb $5, %xmm2, 5(%rdi)3613; SSE4-NEXT: testb $64, %al3614; SSE4-NEXT: je LBB16_143615; SSE4-NEXT: LBB16_13: ## %cond.store113616; SSE4-NEXT: pextrb $6, %xmm2, 6(%rdi)3617; SSE4-NEXT: testb %al, %al3618; SSE4-NEXT: jns LBB16_163619; SSE4-NEXT: LBB16_15: ## %cond.store133620; SSE4-NEXT: pextrb $7, %xmm2, 7(%rdi)3621; SSE4-NEXT: testl $256, %eax ## imm = 0x1003622; SSE4-NEXT: je LBB16_183623; SSE4-NEXT: LBB16_17: ## %cond.store153624; SSE4-NEXT: pextrb $8, %xmm2, 8(%rdi)3625; SSE4-NEXT: testl $512, %eax ## imm = 0x2003626; SSE4-NEXT: je LBB16_203627; SSE4-NEXT: LBB16_19: ## %cond.store173628; SSE4-NEXT: pextrb $9, %xmm2, 9(%rdi)3629; SSE4-NEXT: testl $1024, %eax ## imm = 0x4003630; SSE4-NEXT: je LBB16_223631; SSE4-NEXT: LBB16_21: ## %cond.store193632; SSE4-NEXT: pextrb $10, %xmm2, 10(%rdi)3633; SSE4-NEXT: testl $2048, %eax ## imm = 0x8003634; SSE4-NEXT: je LBB16_243635; SSE4-NEXT: LBB16_23: ## %cond.store213636; SSE4-NEXT: pextrb $11, %xmm2, 11(%rdi)3637; SSE4-NEXT: testl $4096, %eax ## imm = 0x10003638; SSE4-NEXT: je LBB16_263639; SSE4-NEXT: LBB16_25: ## %cond.store233640; SSE4-NEXT: pextrb $12, %xmm2, 12(%rdi)3641; SSE4-NEXT: testl $8192, %eax ## imm = 0x20003642; SSE4-NEXT: je LBB16_283643; SSE4-NEXT: LBB16_27: ## %cond.store253644; SSE4-NEXT: pextrb $13, %xmm2, 13(%rdi)3645; SSE4-NEXT: testl $16384, %eax ## imm = 0x40003646; SSE4-NEXT: je LBB16_303647; SSE4-NEXT: LBB16_29: ## %cond.store273648; SSE4-NEXT: pextrb $14, %xmm2, 14(%rdi)3649; SSE4-NEXT: testw %ax, %ax3650; SSE4-NEXT: jns LBB16_323651; SSE4-NEXT: LBB16_31: ## %cond.store293652; SSE4-NEXT: pextrb $15, %xmm2, 15(%rdi)3653; SSE4-NEXT: testl $65536, %eax ## imm = 0x100003654; SSE4-NEXT: je LBB16_343655; SSE4-NEXT: LBB16_33: ## %cond.store313656; SSE4-NEXT: pextrb $0, %xmm3, 16(%rdi)3657; SSE4-NEXT: testl $131072, %eax ## imm = 0x200003658; SSE4-NEXT: je LBB16_363659; SSE4-NEXT: LBB16_35: ## %cond.store333660; SSE4-NEXT: pextrb $1, %xmm3, 17(%rdi)3661; SSE4-NEXT: testl $262144, %eax ## imm = 0x400003662; SSE4-NEXT: je LBB16_383663; SSE4-NEXT: LBB16_37: ## %cond.store353664; SSE4-NEXT: pextrb $2, %xmm3, 18(%rdi)3665; SSE4-NEXT: testl $524288, %eax ## imm = 0x800003666; SSE4-NEXT: je LBB16_403667; SSE4-NEXT: LBB16_39: ## %cond.store373668; SSE4-NEXT: pextrb $3, %xmm3, 19(%rdi)3669; SSE4-NEXT: testl $1048576, %eax ## imm = 0x1000003670; SSE4-NEXT: je LBB16_423671; SSE4-NEXT: LBB16_41: ## %cond.store393672; SSE4-NEXT: pextrb $4, %xmm3, 20(%rdi)3673; SSE4-NEXT: testl $2097152, %eax ## imm = 0x2000003674; SSE4-NEXT: je LBB16_443675; SSE4-NEXT: LBB16_43: ## %cond.store413676; SSE4-NEXT: pextrb $5, %xmm3, 21(%rdi)3677; SSE4-NEXT: testl $4194304, %eax ## imm = 0x4000003678; SSE4-NEXT: je LBB16_463679; SSE4-NEXT: LBB16_45: ## %cond.store433680; SSE4-NEXT: pextrb $6, %xmm3, 22(%rdi)3681; SSE4-NEXT: testl $8388608, %eax ## imm = 0x8000003682; SSE4-NEXT: je LBB16_483683; SSE4-NEXT: LBB16_47: ## %cond.store453684; SSE4-NEXT: pextrb $7, %xmm3, 23(%rdi)3685; SSE4-NEXT: testl $16777216, %eax ## imm = 0x10000003686; SSE4-NEXT: je LBB16_503687; SSE4-NEXT: LBB16_49: ## %cond.store473688; SSE4-NEXT: pextrb $8, %xmm3, 24(%rdi)3689; SSE4-NEXT: testl $33554432, %eax ## imm = 0x20000003690; SSE4-NEXT: je LBB16_523691; SSE4-NEXT: LBB16_51: ## %cond.store493692; SSE4-NEXT: pextrb $9, %xmm3, 25(%rdi)3693; SSE4-NEXT: testl $67108864, %eax ## imm = 0x40000003694; SSE4-NEXT: je LBB16_543695; SSE4-NEXT: LBB16_53: ## %cond.store513696; SSE4-NEXT: pextrb $10, %xmm3, 26(%rdi)3697; SSE4-NEXT: testl $134217728, %eax ## imm = 0x80000003698; SSE4-NEXT: je LBB16_563699; SSE4-NEXT: LBB16_55: ## %cond.store533700; SSE4-NEXT: pextrb $11, %xmm3, 27(%rdi)3701; SSE4-NEXT: testl $268435456, %eax ## imm = 0x100000003702; SSE4-NEXT: je LBB16_583703; SSE4-NEXT: LBB16_57: ## %cond.store553704; SSE4-NEXT: pextrb $12, %xmm3, 28(%rdi)3705; SSE4-NEXT: testl $536870912, %eax ## imm = 0x200000003706; SSE4-NEXT: je LBB16_603707; SSE4-NEXT: LBB16_59: ## %cond.store573708; SSE4-NEXT: pextrb $13, %xmm3, 29(%rdi)3709; SSE4-NEXT: testl $1073741824, %eax ## imm = 0x400000003710; SSE4-NEXT: je LBB16_623711; SSE4-NEXT: LBB16_61: ## %cond.store593712; SSE4-NEXT: pextrb $14, %xmm3, 30(%rdi)3713; SSE4-NEXT: testl $-2147483648, %eax ## imm = 0x800000003714; SSE4-NEXT: je LBB16_643715; SSE4-NEXT: LBB16_63: ## %cond.store613716; SSE4-NEXT: pextrb $15, %xmm3, 31(%rdi)3717; SSE4-NEXT: retq3718;3719; AVX1-LABEL: store_v32i8_v32i8:3720; AVX1: ## %bb.0:3721; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm23722; AVX1-NEXT: vpcmpeqb %xmm2, %xmm0, %xmm33723; AVX1-NEXT: vpmovmskb %xmm3, %ecx3724; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm03725; AVX1-NEXT: vpcmpeqb %xmm2, %xmm0, %xmm03726; AVX1-NEXT: vpmovmskb %xmm0, %eax3727; AVX1-NEXT: shll $16, %eax3728; AVX1-NEXT: orl %ecx, %eax3729; AVX1-NEXT: testb $1, %al3730; AVX1-NEXT: jne LBB16_13731; AVX1-NEXT: ## %bb.2: ## %else3732; AVX1-NEXT: testb $2, %al3733; AVX1-NEXT: jne LBB16_33734; AVX1-NEXT: LBB16_4: ## %else23735; AVX1-NEXT: testb $4, %al3736; AVX1-NEXT: jne LBB16_53737; AVX1-NEXT: LBB16_6: ## %else43738; AVX1-NEXT: testb $8, %al3739; AVX1-NEXT: jne LBB16_73740; AVX1-NEXT: LBB16_8: ## %else63741; AVX1-NEXT: testb $16, %al3742; AVX1-NEXT: jne LBB16_93743; AVX1-NEXT: LBB16_10: ## %else83744; AVX1-NEXT: testb $32, %al3745; AVX1-NEXT: jne LBB16_113746; AVX1-NEXT: LBB16_12: ## %else103747; AVX1-NEXT: testb $64, %al3748; AVX1-NEXT: jne LBB16_133749; AVX1-NEXT: LBB16_14: ## %else123750; AVX1-NEXT: testb %al, %al3751; AVX1-NEXT: js LBB16_153752; AVX1-NEXT: LBB16_16: ## %else143753; AVX1-NEXT: testl $256, %eax ## imm = 0x1003754; AVX1-NEXT: jne LBB16_173755; AVX1-NEXT: LBB16_18: ## %else163756; AVX1-NEXT: testl $512, %eax ## imm = 0x2003757; AVX1-NEXT: jne LBB16_193758; AVX1-NEXT: LBB16_20: ## %else183759; AVX1-NEXT: testl $1024, %eax ## imm = 0x4003760; AVX1-NEXT: jne LBB16_213761; AVX1-NEXT: LBB16_22: ## %else203762; AVX1-NEXT: testl $2048, %eax ## imm = 0x8003763; AVX1-NEXT: jne LBB16_233764; AVX1-NEXT: LBB16_24: ## %else223765; AVX1-NEXT: testl $4096, %eax ## imm = 0x10003766; AVX1-NEXT: jne LBB16_253767; AVX1-NEXT: LBB16_26: ## %else243768; AVX1-NEXT: testl $8192, %eax ## imm = 0x20003769; AVX1-NEXT: jne LBB16_273770; AVX1-NEXT: LBB16_28: ## %else263771; AVX1-NEXT: testl $16384, %eax ## imm = 0x40003772; AVX1-NEXT: jne LBB16_293773; AVX1-NEXT: LBB16_30: ## %else283774; AVX1-NEXT: testw %ax, %ax3775; AVX1-NEXT: jns LBB16_323776; AVX1-NEXT: LBB16_31: ## %cond.store293777; AVX1-NEXT: vpextrb $15, %xmm1, 15(%rdi)3778; AVX1-NEXT: LBB16_32: ## %else303779; AVX1-NEXT: testl $65536, %eax ## imm = 0x100003780; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm03781; AVX1-NEXT: jne LBB16_333782; AVX1-NEXT: ## %bb.34: ## %else323783; AVX1-NEXT: testl $131072, %eax ## imm = 0x200003784; AVX1-NEXT: jne LBB16_353785; AVX1-NEXT: LBB16_36: ## %else343786; AVX1-NEXT: testl $262144, %eax ## imm = 0x400003787; AVX1-NEXT: jne LBB16_373788; AVX1-NEXT: LBB16_38: ## %else363789; AVX1-NEXT: testl $524288, %eax ## imm = 0x800003790; AVX1-NEXT: jne LBB16_393791; AVX1-NEXT: LBB16_40: ## %else383792; AVX1-NEXT: testl $1048576, %eax ## imm = 0x1000003793; AVX1-NEXT: jne LBB16_413794; AVX1-NEXT: LBB16_42: ## %else403795; AVX1-NEXT: testl $2097152, %eax ## imm = 0x2000003796; AVX1-NEXT: jne LBB16_433797; AVX1-NEXT: LBB16_44: ## %else423798; AVX1-NEXT: testl $4194304, %eax ## imm = 0x4000003799; AVX1-NEXT: jne LBB16_453800; AVX1-NEXT: LBB16_46: ## %else443801; AVX1-NEXT: testl $8388608, %eax ## imm = 0x8000003802; AVX1-NEXT: jne LBB16_473803; AVX1-NEXT: LBB16_48: ## %else463804; AVX1-NEXT: testl $16777216, %eax ## imm = 0x10000003805; AVX1-NEXT: jne LBB16_493806; AVX1-NEXT: LBB16_50: ## %else483807; AVX1-NEXT: testl $33554432, %eax ## imm = 0x20000003808; AVX1-NEXT: jne LBB16_513809; AVX1-NEXT: LBB16_52: ## %else503810; AVX1-NEXT: testl $67108864, %eax ## imm = 0x40000003811; AVX1-NEXT: jne LBB16_533812; AVX1-NEXT: LBB16_54: ## %else523813; AVX1-NEXT: testl $134217728, %eax ## imm = 0x80000003814; AVX1-NEXT: jne LBB16_553815; AVX1-NEXT: LBB16_56: ## %else543816; AVX1-NEXT: testl $268435456, %eax ## imm = 0x100000003817; AVX1-NEXT: jne LBB16_573818; AVX1-NEXT: LBB16_58: ## %else563819; AVX1-NEXT: testl $536870912, %eax ## imm = 0x200000003820; AVX1-NEXT: jne LBB16_593821; AVX1-NEXT: LBB16_60: ## %else583822; AVX1-NEXT: testl $1073741824, %eax ## imm = 0x400000003823; AVX1-NEXT: jne LBB16_613824; AVX1-NEXT: LBB16_62: ## %else603825; AVX1-NEXT: testl $-2147483648, %eax ## imm = 0x800000003826; AVX1-NEXT: jne LBB16_633827; AVX1-NEXT: LBB16_64: ## %else623828; AVX1-NEXT: vzeroupper3829; AVX1-NEXT: retq3830; AVX1-NEXT: LBB16_1: ## %cond.store3831; AVX1-NEXT: vpextrb $0, %xmm1, (%rdi)3832; AVX1-NEXT: testb $2, %al3833; AVX1-NEXT: je LBB16_43834; AVX1-NEXT: LBB16_3: ## %cond.store13835; AVX1-NEXT: vpextrb $1, %xmm1, 1(%rdi)3836; AVX1-NEXT: testb $4, %al3837; AVX1-NEXT: je LBB16_63838; AVX1-NEXT: LBB16_5: ## %cond.store33839; AVX1-NEXT: vpextrb $2, %xmm1, 2(%rdi)3840; AVX1-NEXT: testb $8, %al3841; AVX1-NEXT: je LBB16_83842; AVX1-NEXT: LBB16_7: ## %cond.store53843; AVX1-NEXT: vpextrb $3, %xmm1, 3(%rdi)3844; AVX1-NEXT: testb $16, %al3845; AVX1-NEXT: je LBB16_103846; AVX1-NEXT: LBB16_9: ## %cond.store73847; AVX1-NEXT: vpextrb $4, %xmm1, 4(%rdi)3848; AVX1-NEXT: testb $32, %al3849; AVX1-NEXT: je LBB16_123850; AVX1-NEXT: LBB16_11: ## %cond.store93851; AVX1-NEXT: vpextrb $5, %xmm1, 5(%rdi)3852; AVX1-NEXT: testb $64, %al3853; AVX1-NEXT: je LBB16_143854; AVX1-NEXT: LBB16_13: ## %cond.store113855; AVX1-NEXT: vpextrb $6, %xmm1, 6(%rdi)3856; AVX1-NEXT: testb %al, %al3857; AVX1-NEXT: jns LBB16_163858; AVX1-NEXT: LBB16_15: ## %cond.store133859; AVX1-NEXT: vpextrb $7, %xmm1, 7(%rdi)3860; AVX1-NEXT: testl $256, %eax ## imm = 0x1003861; AVX1-NEXT: je LBB16_183862; AVX1-NEXT: LBB16_17: ## %cond.store153863; AVX1-NEXT: vpextrb $8, %xmm1, 8(%rdi)3864; AVX1-NEXT: testl $512, %eax ## imm = 0x2003865; AVX1-NEXT: je LBB16_203866; AVX1-NEXT: LBB16_19: ## %cond.store173867; AVX1-NEXT: vpextrb $9, %xmm1, 9(%rdi)3868; AVX1-NEXT: testl $1024, %eax ## imm = 0x4003869; AVX1-NEXT: je LBB16_223870; AVX1-NEXT: LBB16_21: ## %cond.store193871; AVX1-NEXT: vpextrb $10, %xmm1, 10(%rdi)3872; AVX1-NEXT: testl $2048, %eax ## imm = 0x8003873; AVX1-NEXT: je LBB16_243874; AVX1-NEXT: LBB16_23: ## %cond.store213875; AVX1-NEXT: vpextrb $11, %xmm1, 11(%rdi)3876; AVX1-NEXT: testl $4096, %eax ## imm = 0x10003877; AVX1-NEXT: je LBB16_263878; AVX1-NEXT: LBB16_25: ## %cond.store233879; AVX1-NEXT: vpextrb $12, %xmm1, 12(%rdi)3880; AVX1-NEXT: testl $8192, %eax ## imm = 0x20003881; AVX1-NEXT: je LBB16_283882; AVX1-NEXT: LBB16_27: ## %cond.store253883; AVX1-NEXT: vpextrb $13, %xmm1, 13(%rdi)3884; AVX1-NEXT: testl $16384, %eax ## imm = 0x40003885; AVX1-NEXT: je LBB16_303886; AVX1-NEXT: LBB16_29: ## %cond.store273887; AVX1-NEXT: vpextrb $14, %xmm1, 14(%rdi)3888; AVX1-NEXT: testw %ax, %ax3889; AVX1-NEXT: js LBB16_313890; AVX1-NEXT: jmp LBB16_323891; AVX1-NEXT: LBB16_33: ## %cond.store313892; AVX1-NEXT: vpextrb $0, %xmm0, 16(%rdi)3893; AVX1-NEXT: testl $131072, %eax ## imm = 0x200003894; AVX1-NEXT: je LBB16_363895; AVX1-NEXT: LBB16_35: ## %cond.store333896; AVX1-NEXT: vpextrb $1, %xmm0, 17(%rdi)3897; AVX1-NEXT: testl $262144, %eax ## imm = 0x400003898; AVX1-NEXT: je LBB16_383899; AVX1-NEXT: LBB16_37: ## %cond.store353900; AVX1-NEXT: vpextrb $2, %xmm0, 18(%rdi)3901; AVX1-NEXT: testl $524288, %eax ## imm = 0x800003902; AVX1-NEXT: je LBB16_403903; AVX1-NEXT: LBB16_39: ## %cond.store373904; AVX1-NEXT: vpextrb $3, %xmm0, 19(%rdi)3905; AVX1-NEXT: testl $1048576, %eax ## imm = 0x1000003906; AVX1-NEXT: je LBB16_423907; AVX1-NEXT: LBB16_41: ## %cond.store393908; AVX1-NEXT: vpextrb $4, %xmm0, 20(%rdi)3909; AVX1-NEXT: testl $2097152, %eax ## imm = 0x2000003910; AVX1-NEXT: je LBB16_443911; AVX1-NEXT: LBB16_43: ## %cond.store413912; AVX1-NEXT: vpextrb $5, %xmm0, 21(%rdi)3913; AVX1-NEXT: testl $4194304, %eax ## imm = 0x4000003914; AVX1-NEXT: je LBB16_463915; AVX1-NEXT: LBB16_45: ## %cond.store433916; AVX1-NEXT: vpextrb $6, %xmm0, 22(%rdi)3917; AVX1-NEXT: testl $8388608, %eax ## imm = 0x8000003918; AVX1-NEXT: je LBB16_483919; AVX1-NEXT: LBB16_47: ## %cond.store453920; AVX1-NEXT: vpextrb $7, %xmm0, 23(%rdi)3921; AVX1-NEXT: testl $16777216, %eax ## imm = 0x10000003922; AVX1-NEXT: je LBB16_503923; AVX1-NEXT: LBB16_49: ## %cond.store473924; AVX1-NEXT: vpextrb $8, %xmm0, 24(%rdi)3925; AVX1-NEXT: testl $33554432, %eax ## imm = 0x20000003926; AVX1-NEXT: je LBB16_523927; AVX1-NEXT: LBB16_51: ## %cond.store493928; AVX1-NEXT: vpextrb $9, %xmm0, 25(%rdi)3929; AVX1-NEXT: testl $67108864, %eax ## imm = 0x40000003930; AVX1-NEXT: je LBB16_543931; AVX1-NEXT: LBB16_53: ## %cond.store513932; AVX1-NEXT: vpextrb $10, %xmm0, 26(%rdi)3933; AVX1-NEXT: testl $134217728, %eax ## imm = 0x80000003934; AVX1-NEXT: je LBB16_563935; AVX1-NEXT: LBB16_55: ## %cond.store533936; AVX1-NEXT: vpextrb $11, %xmm0, 27(%rdi)3937; AVX1-NEXT: testl $268435456, %eax ## imm = 0x100000003938; AVX1-NEXT: je LBB16_583939; AVX1-NEXT: LBB16_57: ## %cond.store553940; AVX1-NEXT: vpextrb $12, %xmm0, 28(%rdi)3941; AVX1-NEXT: testl $536870912, %eax ## imm = 0x200000003942; AVX1-NEXT: je LBB16_603943; AVX1-NEXT: LBB16_59: ## %cond.store573944; AVX1-NEXT: vpextrb $13, %xmm0, 29(%rdi)3945; AVX1-NEXT: testl $1073741824, %eax ## imm = 0x400000003946; AVX1-NEXT: je LBB16_623947; AVX1-NEXT: LBB16_61: ## %cond.store593948; AVX1-NEXT: vpextrb $14, %xmm0, 30(%rdi)3949; AVX1-NEXT: testl $-2147483648, %eax ## imm = 0x800000003950; AVX1-NEXT: je LBB16_643951; AVX1-NEXT: LBB16_63: ## %cond.store613952; AVX1-NEXT: vpextrb $15, %xmm0, 31(%rdi)3953; AVX1-NEXT: vzeroupper3954; AVX1-NEXT: retq3955;3956; AVX2-LABEL: store_v32i8_v32i8:3957; AVX2: ## %bb.0:3958; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm23959; AVX2-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm03960; AVX2-NEXT: vpmovmskb %ymm0, %eax3961; AVX2-NEXT: testb $1, %al3962; AVX2-NEXT: jne LBB16_13963; AVX2-NEXT: ## %bb.2: ## %else3964; AVX2-NEXT: testb $2, %al3965; AVX2-NEXT: jne LBB16_33966; AVX2-NEXT: LBB16_4: ## %else23967; AVX2-NEXT: testb $4, %al3968; AVX2-NEXT: jne LBB16_53969; AVX2-NEXT: LBB16_6: ## %else43970; AVX2-NEXT: testb $8, %al3971; AVX2-NEXT: jne LBB16_73972; AVX2-NEXT: LBB16_8: ## %else63973; AVX2-NEXT: testb $16, %al3974; AVX2-NEXT: jne LBB16_93975; AVX2-NEXT: LBB16_10: ## %else83976; AVX2-NEXT: testb $32, %al3977; AVX2-NEXT: jne LBB16_113978; AVX2-NEXT: LBB16_12: ## %else103979; AVX2-NEXT: testb $64, %al3980; AVX2-NEXT: jne LBB16_133981; AVX2-NEXT: LBB16_14: ## %else123982; AVX2-NEXT: testb %al, %al3983; AVX2-NEXT: js LBB16_153984; AVX2-NEXT: LBB16_16: ## %else143985; AVX2-NEXT: testl $256, %eax ## imm = 0x1003986; AVX2-NEXT: jne LBB16_173987; AVX2-NEXT: LBB16_18: ## %else163988; AVX2-NEXT: testl $512, %eax ## imm = 0x2003989; AVX2-NEXT: jne LBB16_193990; AVX2-NEXT: LBB16_20: ## %else183991; AVX2-NEXT: testl $1024, %eax ## imm = 0x4003992; AVX2-NEXT: jne LBB16_213993; AVX2-NEXT: LBB16_22: ## %else203994; AVX2-NEXT: testl $2048, %eax ## imm = 0x8003995; AVX2-NEXT: jne LBB16_233996; AVX2-NEXT: LBB16_24: ## %else223997; AVX2-NEXT: testl $4096, %eax ## imm = 0x10003998; AVX2-NEXT: jne LBB16_253999; AVX2-NEXT: LBB16_26: ## %else244000; AVX2-NEXT: testl $8192, %eax ## imm = 0x20004001; AVX2-NEXT: jne LBB16_274002; AVX2-NEXT: LBB16_28: ## %else264003; AVX2-NEXT: testl $16384, %eax ## imm = 0x40004004; AVX2-NEXT: jne LBB16_294005; AVX2-NEXT: LBB16_30: ## %else284006; AVX2-NEXT: testw %ax, %ax4007; AVX2-NEXT: jns LBB16_324008; AVX2-NEXT: LBB16_31: ## %cond.store294009; AVX2-NEXT: vpextrb $15, %xmm1, 15(%rdi)4010; AVX2-NEXT: LBB16_32: ## %else304011; AVX2-NEXT: testl $65536, %eax ## imm = 0x100004012; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm04013; AVX2-NEXT: jne LBB16_334014; AVX2-NEXT: ## %bb.34: ## %else324015; AVX2-NEXT: testl $131072, %eax ## imm = 0x200004016; AVX2-NEXT: jne LBB16_354017; AVX2-NEXT: LBB16_36: ## %else344018; AVX2-NEXT: testl $262144, %eax ## imm = 0x400004019; AVX2-NEXT: jne LBB16_374020; AVX2-NEXT: LBB16_38: ## %else364021; AVX2-NEXT: testl $524288, %eax ## imm = 0x800004022; AVX2-NEXT: jne LBB16_394023; AVX2-NEXT: LBB16_40: ## %else384024; AVX2-NEXT: testl $1048576, %eax ## imm = 0x1000004025; AVX2-NEXT: jne LBB16_414026; AVX2-NEXT: LBB16_42: ## %else404027; AVX2-NEXT: testl $2097152, %eax ## imm = 0x2000004028; AVX2-NEXT: jne LBB16_434029; AVX2-NEXT: LBB16_44: ## %else424030; AVX2-NEXT: testl $4194304, %eax ## imm = 0x4000004031; AVX2-NEXT: jne LBB16_454032; AVX2-NEXT: LBB16_46: ## %else444033; AVX2-NEXT: testl $8388608, %eax ## imm = 0x8000004034; AVX2-NEXT: jne LBB16_474035; AVX2-NEXT: LBB16_48: ## %else464036; AVX2-NEXT: testl $16777216, %eax ## imm = 0x10000004037; AVX2-NEXT: jne LBB16_494038; AVX2-NEXT: LBB16_50: ## %else484039; AVX2-NEXT: testl $33554432, %eax ## imm = 0x20000004040; AVX2-NEXT: jne LBB16_514041; AVX2-NEXT: LBB16_52: ## %else504042; AVX2-NEXT: testl $67108864, %eax ## imm = 0x40000004043; AVX2-NEXT: jne LBB16_534044; AVX2-NEXT: LBB16_54: ## %else524045; AVX2-NEXT: testl $134217728, %eax ## imm = 0x80000004046; AVX2-NEXT: jne LBB16_554047; AVX2-NEXT: LBB16_56: ## %else544048; AVX2-NEXT: testl $268435456, %eax ## imm = 0x100000004049; AVX2-NEXT: jne LBB16_574050; AVX2-NEXT: LBB16_58: ## %else564051; AVX2-NEXT: testl $536870912, %eax ## imm = 0x200000004052; AVX2-NEXT: jne LBB16_594053; AVX2-NEXT: LBB16_60: ## %else584054; AVX2-NEXT: testl $1073741824, %eax ## imm = 0x400000004055; AVX2-NEXT: jne LBB16_614056; AVX2-NEXT: LBB16_62: ## %else604057; AVX2-NEXT: testl $-2147483648, %eax ## imm = 0x800000004058; AVX2-NEXT: jne LBB16_634059; AVX2-NEXT: LBB16_64: ## %else624060; AVX2-NEXT: vzeroupper4061; AVX2-NEXT: retq4062; AVX2-NEXT: LBB16_1: ## %cond.store4063; AVX2-NEXT: vpextrb $0, %xmm1, (%rdi)4064; AVX2-NEXT: testb $2, %al4065; AVX2-NEXT: je LBB16_44066; AVX2-NEXT: LBB16_3: ## %cond.store14067; AVX2-NEXT: vpextrb $1, %xmm1, 1(%rdi)4068; AVX2-NEXT: testb $4, %al4069; AVX2-NEXT: je LBB16_64070; AVX2-NEXT: LBB16_5: ## %cond.store34071; AVX2-NEXT: vpextrb $2, %xmm1, 2(%rdi)4072; AVX2-NEXT: testb $8, %al4073; AVX2-NEXT: je LBB16_84074; AVX2-NEXT: LBB16_7: ## %cond.store54075; AVX2-NEXT: vpextrb $3, %xmm1, 3(%rdi)4076; AVX2-NEXT: testb $16, %al4077; AVX2-NEXT: je LBB16_104078; AVX2-NEXT: LBB16_9: ## %cond.store74079; AVX2-NEXT: vpextrb $4, %xmm1, 4(%rdi)4080; AVX2-NEXT: testb $32, %al4081; AVX2-NEXT: je LBB16_124082; AVX2-NEXT: LBB16_11: ## %cond.store94083; AVX2-NEXT: vpextrb $5, %xmm1, 5(%rdi)4084; AVX2-NEXT: testb $64, %al4085; AVX2-NEXT: je LBB16_144086; AVX2-NEXT: LBB16_13: ## %cond.store114087; AVX2-NEXT: vpextrb $6, %xmm1, 6(%rdi)4088; AVX2-NEXT: testb %al, %al4089; AVX2-NEXT: jns LBB16_164090; AVX2-NEXT: LBB16_15: ## %cond.store134091; AVX2-NEXT: vpextrb $7, %xmm1, 7(%rdi)4092; AVX2-NEXT: testl $256, %eax ## imm = 0x1004093; AVX2-NEXT: je LBB16_184094; AVX2-NEXT: LBB16_17: ## %cond.store154095; AVX2-NEXT: vpextrb $8, %xmm1, 8(%rdi)4096; AVX2-NEXT: testl $512, %eax ## imm = 0x2004097; AVX2-NEXT: je LBB16_204098; AVX2-NEXT: LBB16_19: ## %cond.store174099; AVX2-NEXT: vpextrb $9, %xmm1, 9(%rdi)4100; AVX2-NEXT: testl $1024, %eax ## imm = 0x4004101; AVX2-NEXT: je LBB16_224102; AVX2-NEXT: LBB16_21: ## %cond.store194103; AVX2-NEXT: vpextrb $10, %xmm1, 10(%rdi)4104; AVX2-NEXT: testl $2048, %eax ## imm = 0x8004105; AVX2-NEXT: je LBB16_244106; AVX2-NEXT: LBB16_23: ## %cond.store214107; AVX2-NEXT: vpextrb $11, %xmm1, 11(%rdi)4108; AVX2-NEXT: testl $4096, %eax ## imm = 0x10004109; AVX2-NEXT: je LBB16_264110; AVX2-NEXT: LBB16_25: ## %cond.store234111; AVX2-NEXT: vpextrb $12, %xmm1, 12(%rdi)4112; AVX2-NEXT: testl $8192, %eax ## imm = 0x20004113; AVX2-NEXT: je LBB16_284114; AVX2-NEXT: LBB16_27: ## %cond.store254115; AVX2-NEXT: vpextrb $13, %xmm1, 13(%rdi)4116; AVX2-NEXT: testl $16384, %eax ## imm = 0x40004117; AVX2-NEXT: je LBB16_304118; AVX2-NEXT: LBB16_29: ## %cond.store274119; AVX2-NEXT: vpextrb $14, %xmm1, 14(%rdi)4120; AVX2-NEXT: testw %ax, %ax4121; AVX2-NEXT: js LBB16_314122; AVX2-NEXT: jmp LBB16_324123; AVX2-NEXT: LBB16_33: ## %cond.store314124; AVX2-NEXT: vpextrb $0, %xmm0, 16(%rdi)4125; AVX2-NEXT: testl $131072, %eax ## imm = 0x200004126; AVX2-NEXT: je LBB16_364127; AVX2-NEXT: LBB16_35: ## %cond.store334128; AVX2-NEXT: vpextrb $1, %xmm0, 17(%rdi)4129; AVX2-NEXT: testl $262144, %eax ## imm = 0x400004130; AVX2-NEXT: je LBB16_384131; AVX2-NEXT: LBB16_37: ## %cond.store354132; AVX2-NEXT: vpextrb $2, %xmm0, 18(%rdi)4133; AVX2-NEXT: testl $524288, %eax ## imm = 0x800004134; AVX2-NEXT: je LBB16_404135; AVX2-NEXT: LBB16_39: ## %cond.store374136; AVX2-NEXT: vpextrb $3, %xmm0, 19(%rdi)4137; AVX2-NEXT: testl $1048576, %eax ## imm = 0x1000004138; AVX2-NEXT: je LBB16_424139; AVX2-NEXT: LBB16_41: ## %cond.store394140; AVX2-NEXT: vpextrb $4, %xmm0, 20(%rdi)4141; AVX2-NEXT: testl $2097152, %eax ## imm = 0x2000004142; AVX2-NEXT: je LBB16_444143; AVX2-NEXT: LBB16_43: ## %cond.store414144; AVX2-NEXT: vpextrb $5, %xmm0, 21(%rdi)4145; AVX2-NEXT: testl $4194304, %eax ## imm = 0x4000004146; AVX2-NEXT: je LBB16_464147; AVX2-NEXT: LBB16_45: ## %cond.store434148; AVX2-NEXT: vpextrb $6, %xmm0, 22(%rdi)4149; AVX2-NEXT: testl $8388608, %eax ## imm = 0x8000004150; AVX2-NEXT: je LBB16_484151; AVX2-NEXT: LBB16_47: ## %cond.store454152; AVX2-NEXT: vpextrb $7, %xmm0, 23(%rdi)4153; AVX2-NEXT: testl $16777216, %eax ## imm = 0x10000004154; AVX2-NEXT: je LBB16_504155; AVX2-NEXT: LBB16_49: ## %cond.store474156; AVX2-NEXT: vpextrb $8, %xmm0, 24(%rdi)4157; AVX2-NEXT: testl $33554432, %eax ## imm = 0x20000004158; AVX2-NEXT: je LBB16_524159; AVX2-NEXT: LBB16_51: ## %cond.store494160; AVX2-NEXT: vpextrb $9, %xmm0, 25(%rdi)4161; AVX2-NEXT: testl $67108864, %eax ## imm = 0x40000004162; AVX2-NEXT: je LBB16_544163; AVX2-NEXT: LBB16_53: ## %cond.store514164; AVX2-NEXT: vpextrb $10, %xmm0, 26(%rdi)4165; AVX2-NEXT: testl $134217728, %eax ## imm = 0x80000004166; AVX2-NEXT: je LBB16_564167; AVX2-NEXT: LBB16_55: ## %cond.store534168; AVX2-NEXT: vpextrb $11, %xmm0, 27(%rdi)4169; AVX2-NEXT: testl $268435456, %eax ## imm = 0x100000004170; AVX2-NEXT: je LBB16_584171; AVX2-NEXT: LBB16_57: ## %cond.store554172; AVX2-NEXT: vpextrb $12, %xmm0, 28(%rdi)4173; AVX2-NEXT: testl $536870912, %eax ## imm = 0x200000004174; AVX2-NEXT: je LBB16_604175; AVX2-NEXT: LBB16_59: ## %cond.store574176; AVX2-NEXT: vpextrb $13, %xmm0, 29(%rdi)4177; AVX2-NEXT: testl $1073741824, %eax ## imm = 0x400000004178; AVX2-NEXT: je LBB16_624179; AVX2-NEXT: LBB16_61: ## %cond.store594180; AVX2-NEXT: vpextrb $14, %xmm0, 30(%rdi)4181; AVX2-NEXT: testl $-2147483648, %eax ## imm = 0x800000004182; AVX2-NEXT: je LBB16_644183; AVX2-NEXT: LBB16_63: ## %cond.store614184; AVX2-NEXT: vpextrb $15, %xmm0, 31(%rdi)4185; AVX2-NEXT: vzeroupper4186; AVX2-NEXT: retq4187;4188; AVX512F-LABEL: store_v32i8_v32i8:4189; AVX512F: ## %bb.0:4190; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm24191; AVX512F-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm04192; AVX512F-NEXT: vpmovmskb %ymm0, %eax4193; AVX512F-NEXT: testb $1, %al4194; AVX512F-NEXT: jne LBB16_14195; AVX512F-NEXT: ## %bb.2: ## %else4196; AVX512F-NEXT: testb $2, %al4197; AVX512F-NEXT: jne LBB16_34198; AVX512F-NEXT: LBB16_4: ## %else24199; AVX512F-NEXT: testb $4, %al4200; AVX512F-NEXT: jne LBB16_54201; AVX512F-NEXT: LBB16_6: ## %else44202; AVX512F-NEXT: testb $8, %al4203; AVX512F-NEXT: jne LBB16_74204; AVX512F-NEXT: LBB16_8: ## %else64205; AVX512F-NEXT: testb $16, %al4206; AVX512F-NEXT: jne LBB16_94207; AVX512F-NEXT: LBB16_10: ## %else84208; AVX512F-NEXT: testb $32, %al4209; AVX512F-NEXT: jne LBB16_114210; AVX512F-NEXT: LBB16_12: ## %else104211; AVX512F-NEXT: testb $64, %al4212; AVX512F-NEXT: jne LBB16_134213; AVX512F-NEXT: LBB16_14: ## %else124214; AVX512F-NEXT: testb %al, %al4215; AVX512F-NEXT: js LBB16_154216; AVX512F-NEXT: LBB16_16: ## %else144217; AVX512F-NEXT: testl $256, %eax ## imm = 0x1004218; AVX512F-NEXT: jne LBB16_174219; AVX512F-NEXT: LBB16_18: ## %else164220; AVX512F-NEXT: testl $512, %eax ## imm = 0x2004221; AVX512F-NEXT: jne LBB16_194222; AVX512F-NEXT: LBB16_20: ## %else184223; AVX512F-NEXT: testl $1024, %eax ## imm = 0x4004224; AVX512F-NEXT: jne LBB16_214225; AVX512F-NEXT: LBB16_22: ## %else204226; AVX512F-NEXT: testl $2048, %eax ## imm = 0x8004227; AVX512F-NEXT: jne LBB16_234228; AVX512F-NEXT: LBB16_24: ## %else224229; AVX512F-NEXT: testl $4096, %eax ## imm = 0x10004230; AVX512F-NEXT: jne LBB16_254231; AVX512F-NEXT: LBB16_26: ## %else244232; AVX512F-NEXT: testl $8192, %eax ## imm = 0x20004233; AVX512F-NEXT: jne LBB16_274234; AVX512F-NEXT: LBB16_28: ## %else264235; AVX512F-NEXT: testl $16384, %eax ## imm = 0x40004236; AVX512F-NEXT: jne LBB16_294237; AVX512F-NEXT: LBB16_30: ## %else284238; AVX512F-NEXT: testw %ax, %ax4239; AVX512F-NEXT: jns LBB16_324240; AVX512F-NEXT: LBB16_31: ## %cond.store294241; AVX512F-NEXT: vpextrb $15, %xmm1, 15(%rdi)4242; AVX512F-NEXT: LBB16_32: ## %else304243; AVX512F-NEXT: testl $65536, %eax ## imm = 0x100004244; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm04245; AVX512F-NEXT: jne LBB16_334246; AVX512F-NEXT: ## %bb.34: ## %else324247; AVX512F-NEXT: testl $131072, %eax ## imm = 0x200004248; AVX512F-NEXT: jne LBB16_354249; AVX512F-NEXT: LBB16_36: ## %else344250; AVX512F-NEXT: testl $262144, %eax ## imm = 0x400004251; AVX512F-NEXT: jne LBB16_374252; AVX512F-NEXT: LBB16_38: ## %else364253; AVX512F-NEXT: testl $524288, %eax ## imm = 0x800004254; AVX512F-NEXT: jne LBB16_394255; AVX512F-NEXT: LBB16_40: ## %else384256; AVX512F-NEXT: testl $1048576, %eax ## imm = 0x1000004257; AVX512F-NEXT: jne LBB16_414258; AVX512F-NEXT: LBB16_42: ## %else404259; AVX512F-NEXT: testl $2097152, %eax ## imm = 0x2000004260; AVX512F-NEXT: jne LBB16_434261; AVX512F-NEXT: LBB16_44: ## %else424262; AVX512F-NEXT: testl $4194304, %eax ## imm = 0x4000004263; AVX512F-NEXT: jne LBB16_454264; AVX512F-NEXT: LBB16_46: ## %else444265; AVX512F-NEXT: testl $8388608, %eax ## imm = 0x8000004266; AVX512F-NEXT: jne LBB16_474267; AVX512F-NEXT: LBB16_48: ## %else464268; AVX512F-NEXT: testl $16777216, %eax ## imm = 0x10000004269; AVX512F-NEXT: jne LBB16_494270; AVX512F-NEXT: LBB16_50: ## %else484271; AVX512F-NEXT: testl $33554432, %eax ## imm = 0x20000004272; AVX512F-NEXT: jne LBB16_514273; AVX512F-NEXT: LBB16_52: ## %else504274; AVX512F-NEXT: testl $67108864, %eax ## imm = 0x40000004275; AVX512F-NEXT: jne LBB16_534276; AVX512F-NEXT: LBB16_54: ## %else524277; AVX512F-NEXT: testl $134217728, %eax ## imm = 0x80000004278; AVX512F-NEXT: jne LBB16_554279; AVX512F-NEXT: LBB16_56: ## %else544280; AVX512F-NEXT: testl $268435456, %eax ## imm = 0x100000004281; AVX512F-NEXT: jne LBB16_574282; AVX512F-NEXT: LBB16_58: ## %else564283; AVX512F-NEXT: testl $536870912, %eax ## imm = 0x200000004284; AVX512F-NEXT: jne LBB16_594285; AVX512F-NEXT: LBB16_60: ## %else584286; AVX512F-NEXT: testl $1073741824, %eax ## imm = 0x400000004287; AVX512F-NEXT: jne LBB16_614288; AVX512F-NEXT: LBB16_62: ## %else604289; AVX512F-NEXT: testl $-2147483648, %eax ## imm = 0x800000004290; AVX512F-NEXT: jne LBB16_634291; AVX512F-NEXT: LBB16_64: ## %else624292; AVX512F-NEXT: vzeroupper4293; AVX512F-NEXT: retq4294; AVX512F-NEXT: LBB16_1: ## %cond.store4295; AVX512F-NEXT: vpextrb $0, %xmm1, (%rdi)4296; AVX512F-NEXT: testb $2, %al4297; AVX512F-NEXT: je LBB16_44298; AVX512F-NEXT: LBB16_3: ## %cond.store14299; AVX512F-NEXT: vpextrb $1, %xmm1, 1(%rdi)4300; AVX512F-NEXT: testb $4, %al4301; AVX512F-NEXT: je LBB16_64302; AVX512F-NEXT: LBB16_5: ## %cond.store34303; AVX512F-NEXT: vpextrb $2, %xmm1, 2(%rdi)4304; AVX512F-NEXT: testb $8, %al4305; AVX512F-NEXT: je LBB16_84306; AVX512F-NEXT: LBB16_7: ## %cond.store54307; AVX512F-NEXT: vpextrb $3, %xmm1, 3(%rdi)4308; AVX512F-NEXT: testb $16, %al4309; AVX512F-NEXT: je LBB16_104310; AVX512F-NEXT: LBB16_9: ## %cond.store74311; AVX512F-NEXT: vpextrb $4, %xmm1, 4(%rdi)4312; AVX512F-NEXT: testb $32, %al4313; AVX512F-NEXT: je LBB16_124314; AVX512F-NEXT: LBB16_11: ## %cond.store94315; AVX512F-NEXT: vpextrb $5, %xmm1, 5(%rdi)4316; AVX512F-NEXT: testb $64, %al4317; AVX512F-NEXT: je LBB16_144318; AVX512F-NEXT: LBB16_13: ## %cond.store114319; AVX512F-NEXT: vpextrb $6, %xmm1, 6(%rdi)4320; AVX512F-NEXT: testb %al, %al4321; AVX512F-NEXT: jns LBB16_164322; AVX512F-NEXT: LBB16_15: ## %cond.store134323; AVX512F-NEXT: vpextrb $7, %xmm1, 7(%rdi)4324; AVX512F-NEXT: testl $256, %eax ## imm = 0x1004325; AVX512F-NEXT: je LBB16_184326; AVX512F-NEXT: LBB16_17: ## %cond.store154327; AVX512F-NEXT: vpextrb $8, %xmm1, 8(%rdi)4328; AVX512F-NEXT: testl $512, %eax ## imm = 0x2004329; AVX512F-NEXT: je LBB16_204330; AVX512F-NEXT: LBB16_19: ## %cond.store174331; AVX512F-NEXT: vpextrb $9, %xmm1, 9(%rdi)4332; AVX512F-NEXT: testl $1024, %eax ## imm = 0x4004333; AVX512F-NEXT: je LBB16_224334; AVX512F-NEXT: LBB16_21: ## %cond.store194335; AVX512F-NEXT: vpextrb $10, %xmm1, 10(%rdi)4336; AVX512F-NEXT: testl $2048, %eax ## imm = 0x8004337; AVX512F-NEXT: je LBB16_244338; AVX512F-NEXT: LBB16_23: ## %cond.store214339; AVX512F-NEXT: vpextrb $11, %xmm1, 11(%rdi)4340; AVX512F-NEXT: testl $4096, %eax ## imm = 0x10004341; AVX512F-NEXT: je LBB16_264342; AVX512F-NEXT: LBB16_25: ## %cond.store234343; AVX512F-NEXT: vpextrb $12, %xmm1, 12(%rdi)4344; AVX512F-NEXT: testl $8192, %eax ## imm = 0x20004345; AVX512F-NEXT: je LBB16_284346; AVX512F-NEXT: LBB16_27: ## %cond.store254347; AVX512F-NEXT: vpextrb $13, %xmm1, 13(%rdi)4348; AVX512F-NEXT: testl $16384, %eax ## imm = 0x40004349; AVX512F-NEXT: je LBB16_304350; AVX512F-NEXT: LBB16_29: ## %cond.store274351; AVX512F-NEXT: vpextrb $14, %xmm1, 14(%rdi)4352; AVX512F-NEXT: testw %ax, %ax4353; AVX512F-NEXT: js LBB16_314354; AVX512F-NEXT: jmp LBB16_324355; AVX512F-NEXT: LBB16_33: ## %cond.store314356; AVX512F-NEXT: vpextrb $0, %xmm0, 16(%rdi)4357; AVX512F-NEXT: testl $131072, %eax ## imm = 0x200004358; AVX512F-NEXT: je LBB16_364359; AVX512F-NEXT: LBB16_35: ## %cond.store334360; AVX512F-NEXT: vpextrb $1, %xmm0, 17(%rdi)4361; AVX512F-NEXT: testl $262144, %eax ## imm = 0x400004362; AVX512F-NEXT: je LBB16_384363; AVX512F-NEXT: LBB16_37: ## %cond.store354364; AVX512F-NEXT: vpextrb $2, %xmm0, 18(%rdi)4365; AVX512F-NEXT: testl $524288, %eax ## imm = 0x800004366; AVX512F-NEXT: je LBB16_404367; AVX512F-NEXT: LBB16_39: ## %cond.store374368; AVX512F-NEXT: vpextrb $3, %xmm0, 19(%rdi)4369; AVX512F-NEXT: testl $1048576, %eax ## imm = 0x1000004370; AVX512F-NEXT: je LBB16_424371; AVX512F-NEXT: LBB16_41: ## %cond.store394372; AVX512F-NEXT: vpextrb $4, %xmm0, 20(%rdi)4373; AVX512F-NEXT: testl $2097152, %eax ## imm = 0x2000004374; AVX512F-NEXT: je LBB16_444375; AVX512F-NEXT: LBB16_43: ## %cond.store414376; AVX512F-NEXT: vpextrb $5, %xmm0, 21(%rdi)4377; AVX512F-NEXT: testl $4194304, %eax ## imm = 0x4000004378; AVX512F-NEXT: je LBB16_464379; AVX512F-NEXT: LBB16_45: ## %cond.store434380; AVX512F-NEXT: vpextrb $6, %xmm0, 22(%rdi)4381; AVX512F-NEXT: testl $8388608, %eax ## imm = 0x8000004382; AVX512F-NEXT: je LBB16_484383; AVX512F-NEXT: LBB16_47: ## %cond.store454384; AVX512F-NEXT: vpextrb $7, %xmm0, 23(%rdi)4385; AVX512F-NEXT: testl $16777216, %eax ## imm = 0x10000004386; AVX512F-NEXT: je LBB16_504387; AVX512F-NEXT: LBB16_49: ## %cond.store474388; AVX512F-NEXT: vpextrb $8, %xmm0, 24(%rdi)4389; AVX512F-NEXT: testl $33554432, %eax ## imm = 0x20000004390; AVX512F-NEXT: je LBB16_524391; AVX512F-NEXT: LBB16_51: ## %cond.store494392; AVX512F-NEXT: vpextrb $9, %xmm0, 25(%rdi)4393; AVX512F-NEXT: testl $67108864, %eax ## imm = 0x40000004394; AVX512F-NEXT: je LBB16_544395; AVX512F-NEXT: LBB16_53: ## %cond.store514396; AVX512F-NEXT: vpextrb $10, %xmm0, 26(%rdi)4397; AVX512F-NEXT: testl $134217728, %eax ## imm = 0x80000004398; AVX512F-NEXT: je LBB16_564399; AVX512F-NEXT: LBB16_55: ## %cond.store534400; AVX512F-NEXT: vpextrb $11, %xmm0, 27(%rdi)4401; AVX512F-NEXT: testl $268435456, %eax ## imm = 0x100000004402; AVX512F-NEXT: je LBB16_584403; AVX512F-NEXT: LBB16_57: ## %cond.store554404; AVX512F-NEXT: vpextrb $12, %xmm0, 28(%rdi)4405; AVX512F-NEXT: testl $536870912, %eax ## imm = 0x200000004406; AVX512F-NEXT: je LBB16_604407; AVX512F-NEXT: LBB16_59: ## %cond.store574408; AVX512F-NEXT: vpextrb $13, %xmm0, 29(%rdi)4409; AVX512F-NEXT: testl $1073741824, %eax ## imm = 0x400000004410; AVX512F-NEXT: je LBB16_624411; AVX512F-NEXT: LBB16_61: ## %cond.store594412; AVX512F-NEXT: vpextrb $14, %xmm0, 30(%rdi)4413; AVX512F-NEXT: testl $-2147483648, %eax ## imm = 0x800000004414; AVX512F-NEXT: je LBB16_644415; AVX512F-NEXT: LBB16_63: ## %cond.store614416; AVX512F-NEXT: vpextrb $15, %xmm0, 31(%rdi)4417; AVX512F-NEXT: vzeroupper4418; AVX512F-NEXT: retq4419;4420; AVX512VLDQ-LABEL: store_v32i8_v32i8:4421; AVX512VLDQ: ## %bb.0:4422; AVX512VLDQ-NEXT: vpxor %xmm2, %xmm2, %xmm24423; AVX512VLDQ-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm04424; AVX512VLDQ-NEXT: vpmovmskb %ymm0, %eax4425; AVX512VLDQ-NEXT: testb $1, %al4426; AVX512VLDQ-NEXT: jne LBB16_14427; AVX512VLDQ-NEXT: ## %bb.2: ## %else4428; AVX512VLDQ-NEXT: testb $2, %al4429; AVX512VLDQ-NEXT: jne LBB16_34430; AVX512VLDQ-NEXT: LBB16_4: ## %else24431; AVX512VLDQ-NEXT: testb $4, %al4432; AVX512VLDQ-NEXT: jne LBB16_54433; AVX512VLDQ-NEXT: LBB16_6: ## %else44434; AVX512VLDQ-NEXT: testb $8, %al4435; AVX512VLDQ-NEXT: jne LBB16_74436; AVX512VLDQ-NEXT: LBB16_8: ## %else64437; AVX512VLDQ-NEXT: testb $16, %al4438; AVX512VLDQ-NEXT: jne LBB16_94439; AVX512VLDQ-NEXT: LBB16_10: ## %else84440; AVX512VLDQ-NEXT: testb $32, %al4441; AVX512VLDQ-NEXT: jne LBB16_114442; AVX512VLDQ-NEXT: LBB16_12: ## %else104443; AVX512VLDQ-NEXT: testb $64, %al4444; AVX512VLDQ-NEXT: jne LBB16_134445; AVX512VLDQ-NEXT: LBB16_14: ## %else124446; AVX512VLDQ-NEXT: testb %al, %al4447; AVX512VLDQ-NEXT: js LBB16_154448; AVX512VLDQ-NEXT: LBB16_16: ## %else144449; AVX512VLDQ-NEXT: testl $256, %eax ## imm = 0x1004450; AVX512VLDQ-NEXT: jne LBB16_174451; AVX512VLDQ-NEXT: LBB16_18: ## %else164452; AVX512VLDQ-NEXT: testl $512, %eax ## imm = 0x2004453; AVX512VLDQ-NEXT: jne LBB16_194454; AVX512VLDQ-NEXT: LBB16_20: ## %else184455; AVX512VLDQ-NEXT: testl $1024, %eax ## imm = 0x4004456; AVX512VLDQ-NEXT: jne LBB16_214457; AVX512VLDQ-NEXT: LBB16_22: ## %else204458; AVX512VLDQ-NEXT: testl $2048, %eax ## imm = 0x8004459; AVX512VLDQ-NEXT: jne LBB16_234460; AVX512VLDQ-NEXT: LBB16_24: ## %else224461; AVX512VLDQ-NEXT: testl $4096, %eax ## imm = 0x10004462; AVX512VLDQ-NEXT: jne LBB16_254463; AVX512VLDQ-NEXT: LBB16_26: ## %else244464; AVX512VLDQ-NEXT: testl $8192, %eax ## imm = 0x20004465; AVX512VLDQ-NEXT: jne LBB16_274466; AVX512VLDQ-NEXT: LBB16_28: ## %else264467; AVX512VLDQ-NEXT: testl $16384, %eax ## imm = 0x40004468; AVX512VLDQ-NEXT: jne LBB16_294469; AVX512VLDQ-NEXT: LBB16_30: ## %else284470; AVX512VLDQ-NEXT: testw %ax, %ax4471; AVX512VLDQ-NEXT: jns LBB16_324472; AVX512VLDQ-NEXT: LBB16_31: ## %cond.store294473; AVX512VLDQ-NEXT: vpextrb $15, %xmm1, 15(%rdi)4474; AVX512VLDQ-NEXT: LBB16_32: ## %else304475; AVX512VLDQ-NEXT: testl $65536, %eax ## imm = 0x100004476; AVX512VLDQ-NEXT: vextracti128 $1, %ymm1, %xmm04477; AVX512VLDQ-NEXT: jne LBB16_334478; AVX512VLDQ-NEXT: ## %bb.34: ## %else324479; AVX512VLDQ-NEXT: testl $131072, %eax ## imm = 0x200004480; AVX512VLDQ-NEXT: jne LBB16_354481; AVX512VLDQ-NEXT: LBB16_36: ## %else344482; AVX512VLDQ-NEXT: testl $262144, %eax ## imm = 0x400004483; AVX512VLDQ-NEXT: jne LBB16_374484; AVX512VLDQ-NEXT: LBB16_38: ## %else364485; AVX512VLDQ-NEXT: testl $524288, %eax ## imm = 0x800004486; AVX512VLDQ-NEXT: jne LBB16_394487; AVX512VLDQ-NEXT: LBB16_40: ## %else384488; AVX512VLDQ-NEXT: testl $1048576, %eax ## imm = 0x1000004489; AVX512VLDQ-NEXT: jne LBB16_414490; AVX512VLDQ-NEXT: LBB16_42: ## %else404491; AVX512VLDQ-NEXT: testl $2097152, %eax ## imm = 0x2000004492; AVX512VLDQ-NEXT: jne LBB16_434493; AVX512VLDQ-NEXT: LBB16_44: ## %else424494; AVX512VLDQ-NEXT: testl $4194304, %eax ## imm = 0x4000004495; AVX512VLDQ-NEXT: jne LBB16_454496; AVX512VLDQ-NEXT: LBB16_46: ## %else444497; AVX512VLDQ-NEXT: testl $8388608, %eax ## imm = 0x8000004498; AVX512VLDQ-NEXT: jne LBB16_474499; AVX512VLDQ-NEXT: LBB16_48: ## %else464500; AVX512VLDQ-NEXT: testl $16777216, %eax ## imm = 0x10000004501; AVX512VLDQ-NEXT: jne LBB16_494502; AVX512VLDQ-NEXT: LBB16_50: ## %else484503; AVX512VLDQ-NEXT: testl $33554432, %eax ## imm = 0x20000004504; AVX512VLDQ-NEXT: jne LBB16_514505; AVX512VLDQ-NEXT: LBB16_52: ## %else504506; AVX512VLDQ-NEXT: testl $67108864, %eax ## imm = 0x40000004507; AVX512VLDQ-NEXT: jne LBB16_534508; AVX512VLDQ-NEXT: LBB16_54: ## %else524509; AVX512VLDQ-NEXT: testl $134217728, %eax ## imm = 0x80000004510; AVX512VLDQ-NEXT: jne LBB16_554511; AVX512VLDQ-NEXT: LBB16_56: ## %else544512; AVX512VLDQ-NEXT: testl $268435456, %eax ## imm = 0x100000004513; AVX512VLDQ-NEXT: jne LBB16_574514; AVX512VLDQ-NEXT: LBB16_58: ## %else564515; AVX512VLDQ-NEXT: testl $536870912, %eax ## imm = 0x200000004516; AVX512VLDQ-NEXT: jne LBB16_594517; AVX512VLDQ-NEXT: LBB16_60: ## %else584518; AVX512VLDQ-NEXT: testl $1073741824, %eax ## imm = 0x400000004519; AVX512VLDQ-NEXT: jne LBB16_614520; AVX512VLDQ-NEXT: LBB16_62: ## %else604521; AVX512VLDQ-NEXT: testl $-2147483648, %eax ## imm = 0x800000004522; AVX512VLDQ-NEXT: jne LBB16_634523; AVX512VLDQ-NEXT: LBB16_64: ## %else624524; AVX512VLDQ-NEXT: vzeroupper4525; AVX512VLDQ-NEXT: retq4526; AVX512VLDQ-NEXT: LBB16_1: ## %cond.store4527; AVX512VLDQ-NEXT: vpextrb $0, %xmm1, (%rdi)4528; AVX512VLDQ-NEXT: testb $2, %al4529; AVX512VLDQ-NEXT: je LBB16_44530; AVX512VLDQ-NEXT: LBB16_3: ## %cond.store14531; AVX512VLDQ-NEXT: vpextrb $1, %xmm1, 1(%rdi)4532; AVX512VLDQ-NEXT: testb $4, %al4533; AVX512VLDQ-NEXT: je LBB16_64534; AVX512VLDQ-NEXT: LBB16_5: ## %cond.store34535; AVX512VLDQ-NEXT: vpextrb $2, %xmm1, 2(%rdi)4536; AVX512VLDQ-NEXT: testb $8, %al4537; AVX512VLDQ-NEXT: je LBB16_84538; AVX512VLDQ-NEXT: LBB16_7: ## %cond.store54539; AVX512VLDQ-NEXT: vpextrb $3, %xmm1, 3(%rdi)4540; AVX512VLDQ-NEXT: testb $16, %al4541; AVX512VLDQ-NEXT: je LBB16_104542; AVX512VLDQ-NEXT: LBB16_9: ## %cond.store74543; AVX512VLDQ-NEXT: vpextrb $4, %xmm1, 4(%rdi)4544; AVX512VLDQ-NEXT: testb $32, %al4545; AVX512VLDQ-NEXT: je LBB16_124546; AVX512VLDQ-NEXT: LBB16_11: ## %cond.store94547; AVX512VLDQ-NEXT: vpextrb $5, %xmm1, 5(%rdi)4548; AVX512VLDQ-NEXT: testb $64, %al4549; AVX512VLDQ-NEXT: je LBB16_144550; AVX512VLDQ-NEXT: LBB16_13: ## %cond.store114551; AVX512VLDQ-NEXT: vpextrb $6, %xmm1, 6(%rdi)4552; AVX512VLDQ-NEXT: testb %al, %al4553; AVX512VLDQ-NEXT: jns LBB16_164554; AVX512VLDQ-NEXT: LBB16_15: ## %cond.store134555; AVX512VLDQ-NEXT: vpextrb $7, %xmm1, 7(%rdi)4556; AVX512VLDQ-NEXT: testl $256, %eax ## imm = 0x1004557; AVX512VLDQ-NEXT: je LBB16_184558; AVX512VLDQ-NEXT: LBB16_17: ## %cond.store154559; AVX512VLDQ-NEXT: vpextrb $8, %xmm1, 8(%rdi)4560; AVX512VLDQ-NEXT: testl $512, %eax ## imm = 0x2004561; AVX512VLDQ-NEXT: je LBB16_204562; AVX512VLDQ-NEXT: LBB16_19: ## %cond.store174563; AVX512VLDQ-NEXT: vpextrb $9, %xmm1, 9(%rdi)4564; AVX512VLDQ-NEXT: testl $1024, %eax ## imm = 0x4004565; AVX512VLDQ-NEXT: je LBB16_224566; AVX512VLDQ-NEXT: LBB16_21: ## %cond.store194567; AVX512VLDQ-NEXT: vpextrb $10, %xmm1, 10(%rdi)4568; AVX512VLDQ-NEXT: testl $2048, %eax ## imm = 0x8004569; AVX512VLDQ-NEXT: je LBB16_244570; AVX512VLDQ-NEXT: LBB16_23: ## %cond.store214571; AVX512VLDQ-NEXT: vpextrb $11, %xmm1, 11(%rdi)4572; AVX512VLDQ-NEXT: testl $4096, %eax ## imm = 0x10004573; AVX512VLDQ-NEXT: je LBB16_264574; AVX512VLDQ-NEXT: LBB16_25: ## %cond.store234575; AVX512VLDQ-NEXT: vpextrb $12, %xmm1, 12(%rdi)4576; AVX512VLDQ-NEXT: testl $8192, %eax ## imm = 0x20004577; AVX512VLDQ-NEXT: je LBB16_284578; AVX512VLDQ-NEXT: LBB16_27: ## %cond.store254579; AVX512VLDQ-NEXT: vpextrb $13, %xmm1, 13(%rdi)4580; AVX512VLDQ-NEXT: testl $16384, %eax ## imm = 0x40004581; AVX512VLDQ-NEXT: je LBB16_304582; AVX512VLDQ-NEXT: LBB16_29: ## %cond.store274583; AVX512VLDQ-NEXT: vpextrb $14, %xmm1, 14(%rdi)4584; AVX512VLDQ-NEXT: testw %ax, %ax4585; AVX512VLDQ-NEXT: js LBB16_314586; AVX512VLDQ-NEXT: jmp LBB16_324587; AVX512VLDQ-NEXT: LBB16_33: ## %cond.store314588; AVX512VLDQ-NEXT: vpextrb $0, %xmm0, 16(%rdi)4589; AVX512VLDQ-NEXT: testl $131072, %eax ## imm = 0x200004590; AVX512VLDQ-NEXT: je LBB16_364591; AVX512VLDQ-NEXT: LBB16_35: ## %cond.store334592; AVX512VLDQ-NEXT: vpextrb $1, %xmm0, 17(%rdi)4593; AVX512VLDQ-NEXT: testl $262144, %eax ## imm = 0x400004594; AVX512VLDQ-NEXT: je LBB16_384595; AVX512VLDQ-NEXT: LBB16_37: ## %cond.store354596; AVX512VLDQ-NEXT: vpextrb $2, %xmm0, 18(%rdi)4597; AVX512VLDQ-NEXT: testl $524288, %eax ## imm = 0x800004598; AVX512VLDQ-NEXT: je LBB16_404599; AVX512VLDQ-NEXT: LBB16_39: ## %cond.store374600; AVX512VLDQ-NEXT: vpextrb $3, %xmm0, 19(%rdi)4601; AVX512VLDQ-NEXT: testl $1048576, %eax ## imm = 0x1000004602; AVX512VLDQ-NEXT: je LBB16_424603; AVX512VLDQ-NEXT: LBB16_41: ## %cond.store394604; AVX512VLDQ-NEXT: vpextrb $4, %xmm0, 20(%rdi)4605; AVX512VLDQ-NEXT: testl $2097152, %eax ## imm = 0x2000004606; AVX512VLDQ-NEXT: je LBB16_444607; AVX512VLDQ-NEXT: LBB16_43: ## %cond.store414608; AVX512VLDQ-NEXT: vpextrb $5, %xmm0, 21(%rdi)4609; AVX512VLDQ-NEXT: testl $4194304, %eax ## imm = 0x4000004610; AVX512VLDQ-NEXT: je LBB16_464611; AVX512VLDQ-NEXT: LBB16_45: ## %cond.store434612; AVX512VLDQ-NEXT: vpextrb $6, %xmm0, 22(%rdi)4613; AVX512VLDQ-NEXT: testl $8388608, %eax ## imm = 0x8000004614; AVX512VLDQ-NEXT: je LBB16_484615; AVX512VLDQ-NEXT: LBB16_47: ## %cond.store454616; AVX512VLDQ-NEXT: vpextrb $7, %xmm0, 23(%rdi)4617; AVX512VLDQ-NEXT: testl $16777216, %eax ## imm = 0x10000004618; AVX512VLDQ-NEXT: je LBB16_504619; AVX512VLDQ-NEXT: LBB16_49: ## %cond.store474620; AVX512VLDQ-NEXT: vpextrb $8, %xmm0, 24(%rdi)4621; AVX512VLDQ-NEXT: testl $33554432, %eax ## imm = 0x20000004622; AVX512VLDQ-NEXT: je LBB16_524623; AVX512VLDQ-NEXT: LBB16_51: ## %cond.store494624; AVX512VLDQ-NEXT: vpextrb $9, %xmm0, 25(%rdi)4625; AVX512VLDQ-NEXT: testl $67108864, %eax ## imm = 0x40000004626; AVX512VLDQ-NEXT: je LBB16_544627; AVX512VLDQ-NEXT: LBB16_53: ## %cond.store514628; AVX512VLDQ-NEXT: vpextrb $10, %xmm0, 26(%rdi)4629; AVX512VLDQ-NEXT: testl $134217728, %eax ## imm = 0x80000004630; AVX512VLDQ-NEXT: je LBB16_564631; AVX512VLDQ-NEXT: LBB16_55: ## %cond.store534632; AVX512VLDQ-NEXT: vpextrb $11, %xmm0, 27(%rdi)4633; AVX512VLDQ-NEXT: testl $268435456, %eax ## imm = 0x100000004634; AVX512VLDQ-NEXT: je LBB16_584635; AVX512VLDQ-NEXT: LBB16_57: ## %cond.store554636; AVX512VLDQ-NEXT: vpextrb $12, %xmm0, 28(%rdi)4637; AVX512VLDQ-NEXT: testl $536870912, %eax ## imm = 0x200000004638; AVX512VLDQ-NEXT: je LBB16_604639; AVX512VLDQ-NEXT: LBB16_59: ## %cond.store574640; AVX512VLDQ-NEXT: vpextrb $13, %xmm0, 29(%rdi)4641; AVX512VLDQ-NEXT: testl $1073741824, %eax ## imm = 0x400000004642; AVX512VLDQ-NEXT: je LBB16_624643; AVX512VLDQ-NEXT: LBB16_61: ## %cond.store594644; AVX512VLDQ-NEXT: vpextrb $14, %xmm0, 30(%rdi)4645; AVX512VLDQ-NEXT: testl $-2147483648, %eax ## imm = 0x800000004646; AVX512VLDQ-NEXT: je LBB16_644647; AVX512VLDQ-NEXT: LBB16_63: ## %cond.store614648; AVX512VLDQ-NEXT: vpextrb $15, %xmm0, 31(%rdi)4649; AVX512VLDQ-NEXT: vzeroupper4650; AVX512VLDQ-NEXT: retq4651;4652; AVX512VLBW-LABEL: store_v32i8_v32i8:4653; AVX512VLBW: ## %bb.0:4654; AVX512VLBW-NEXT: vptestnmb %ymm0, %ymm0, %k14655; AVX512VLBW-NEXT: vmovdqu8 %ymm1, (%rdi) {%k1}4656; AVX512VLBW-NEXT: vzeroupper4657; AVX512VLBW-NEXT: retq4658;4659; X86-AVX512-LABEL: store_v32i8_v32i8:4660; X86-AVX512: ## %bb.0:4661; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax4662; X86-AVX512-NEXT: vptestnmb %ymm0, %ymm0, %k14663; X86-AVX512-NEXT: vmovdqu8 %ymm1, (%eax) {%k1}4664; X86-AVX512-NEXT: vzeroupper4665; X86-AVX512-NEXT: retl4666 %mask = icmp eq <32 x i8> %trigger, zeroinitializer4667 call void @llvm.masked.store.v32i8.p0(<32 x i8> %val, ptr %addr, i32 4, <32 x i1> %mask)4668 ret void4669}4670 4671;;; Stores with Constant Masks4672 4673define void @mstore_constmask_v4i32_v4i32(<4 x i32> %trigger, ptr %addr, <4 x i32> %val) nounwind {4674; SSE-LABEL: mstore_constmask_v4i32_v4i32:4675; SSE: ## %bb.0:4676; SSE-NEXT: movups %xmm1, (%rdi)4677; SSE-NEXT: retq4678;4679; AVX-LABEL: mstore_constmask_v4i32_v4i32:4680; AVX: ## %bb.0:4681; AVX-NEXT: vmovups %xmm1, (%rdi)4682; AVX-NEXT: retq4683;4684; X86-AVX512-LABEL: mstore_constmask_v4i32_v4i32:4685; X86-AVX512: ## %bb.0:4686; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax4687; X86-AVX512-NEXT: vmovups %xmm1, (%eax)4688; X86-AVX512-NEXT: retl4689 %mask = icmp eq <4 x i32> %trigger, zeroinitializer4690 call void @llvm.masked.store.v4i32.p0(<4 x i32> %val, ptr %addr, i32 4, <4 x i1><i1 true, i1 true, i1 true, i1 true>)4691 ret void4692}4693 4694; Make sure we are able to detect all ones constant mask after type legalization4695; to avoid masked stores.4696define void @mstore_constmask_allones_split(<16 x i64> %trigger, ptr %addr, <16 x i64> %val) nounwind {4697; SSE2-LABEL: mstore_constmask_allones_split:4698; SSE2: ## %bb.0:4699; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm04700; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm14701; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm24702; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm34703; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm44704; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm54705; SSE2-NEXT: movq %xmm5, (%rdi)4706; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]4707; SSE2-NEXT: movq %xmm5, 8(%rdi)4708; SSE2-NEXT: pshufd {{.*#+}} xmm5 = mem[2,3,2,3]4709; SSE2-NEXT: movq %xmm5, 24(%rdi)4710; SSE2-NEXT: movq {{[0-9]+}}(%rsp), %rax4711; SSE2-NEXT: movq %rax, 32(%rdi)4712; SSE2-NEXT: movq %xmm4, 48(%rdi)4713; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]4714; SSE2-NEXT: movq %xmm4, 56(%rdi)4715; SSE2-NEXT: movq %xmm3, 64(%rdi)4716; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]4717; SSE2-NEXT: movq %xmm3, 72(%rdi)4718; SSE2-NEXT: movq %xmm2, 80(%rdi)4719; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]4720; SSE2-NEXT: movq %xmm2, 88(%rdi)4721; SSE2-NEXT: movq %xmm1, 96(%rdi)4722; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]4723; SSE2-NEXT: movq %xmm1, 104(%rdi)4724; SSE2-NEXT: movq %xmm0, 112(%rdi)4725; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]4726; SSE2-NEXT: movq %xmm0, 120(%rdi)4727; SSE2-NEXT: retq4728;4729; SSE4-LABEL: mstore_constmask_allones_split:4730; SSE4: ## %bb.0:4731; SSE4-NEXT: movaps {{[0-9]+}}(%rsp), %xmm04732; SSE4-NEXT: movaps {{[0-9]+}}(%rsp), %xmm14733; SSE4-NEXT: movaps {{[0-9]+}}(%rsp), %xmm24734; SSE4-NEXT: movaps {{[0-9]+}}(%rsp), %xmm34735; SSE4-NEXT: movaps {{[0-9]+}}(%rsp), %xmm44736; SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm54737; SSE4-NEXT: movaps {{[0-9]+}}(%rsp), %xmm64738; SSE4-NEXT: movups %xmm6, (%rdi)4739; SSE4-NEXT: palignr {{.*#+}} xmm5 = mem[8,9,10,11,12,13,14,15],xmm5[0,1,2,3,4,5,6,7]4740; SSE4-NEXT: movdqu %xmm5, 24(%rdi)4741; SSE4-NEXT: movups %xmm4, 48(%rdi)4742; SSE4-NEXT: movups %xmm3, 64(%rdi)4743; SSE4-NEXT: movups %xmm2, 80(%rdi)4744; SSE4-NEXT: movups %xmm1, 96(%rdi)4745; SSE4-NEXT: movups %xmm0, 112(%rdi)4746; SSE4-NEXT: retq4747;4748; AVX1-LABEL: mstore_constmask_allones_split:4749; AVX1: ## %bb.0:4750; AVX1-NEXT: vmovapd {{.*#+}} ymm0 = [18446744073709551615,0,18446744073709551615,18446744073709551615]4751; AVX1-NEXT: vmaskmovpd %ymm5, %ymm0, 32(%rdi)4752; AVX1-NEXT: vmovapd {{.*#+}} ymm0 = [18446744073709551615,18446744073709551615,0,18446744073709551615]4753; AVX1-NEXT: vmaskmovpd %ymm4, %ymm0, (%rdi)4754; AVX1-NEXT: vmovups %ymm7, 96(%rdi)4755; AVX1-NEXT: vmovups %ymm6, 64(%rdi)4756; AVX1-NEXT: vzeroupper4757; AVX1-NEXT: retq4758;4759; AVX2-LABEL: mstore_constmask_allones_split:4760; AVX2: ## %bb.0:4761; AVX2-NEXT: vpmovsxbq {{.*#+}} ymm0 = [18446744073709551615,0,18446744073709551615,18446744073709551615]4762; AVX2-NEXT: vpmaskmovq %ymm5, %ymm0, 32(%rdi)4763; AVX2-NEXT: vpmovsxbq {{.*#+}} ymm0 = [18446744073709551615,18446744073709551615,0,18446744073709551615]4764; AVX2-NEXT: vpmaskmovq %ymm4, %ymm0, (%rdi)4765; AVX2-NEXT: vmovups %ymm7, 96(%rdi)4766; AVX2-NEXT: vmovups %ymm6, 64(%rdi)4767; AVX2-NEXT: vzeroupper4768; AVX2-NEXT: retq4769;4770; AVX512F-LABEL: mstore_constmask_allones_split:4771; AVX512F: ## %bb.0:4772; AVX512F-NEXT: movb $-37, %al4773; AVX512F-NEXT: kmovw %eax, %k14774; AVX512F-NEXT: vmovdqu64 %zmm2, (%rdi) {%k1}4775; AVX512F-NEXT: vmovups %zmm3, 64(%rdi)4776; AVX512F-NEXT: vzeroupper4777; AVX512F-NEXT: retq4778;4779; AVX512VLDQ-LABEL: mstore_constmask_allones_split:4780; AVX512VLDQ: ## %bb.0:4781; AVX512VLDQ-NEXT: movb $-37, %al4782; AVX512VLDQ-NEXT: kmovw %eax, %k14783; AVX512VLDQ-NEXT: vmovdqu64 %zmm2, (%rdi) {%k1}4784; AVX512VLDQ-NEXT: vmovups %zmm3, 64(%rdi)4785; AVX512VLDQ-NEXT: vzeroupper4786; AVX512VLDQ-NEXT: retq4787;4788; AVX512VLBW-LABEL: mstore_constmask_allones_split:4789; AVX512VLBW: ## %bb.0:4790; AVX512VLBW-NEXT: movb $-37, %al4791; AVX512VLBW-NEXT: kmovd %eax, %k14792; AVX512VLBW-NEXT: vmovdqu64 %zmm2, (%rdi) {%k1}4793; AVX512VLBW-NEXT: vmovups %zmm3, 64(%rdi)4794; AVX512VLBW-NEXT: vzeroupper4795; AVX512VLBW-NEXT: retq4796;4797; X86-AVX512-LABEL: mstore_constmask_allones_split:4798; X86-AVX512: ## %bb.0:4799; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax4800; X86-AVX512-NEXT: movb $-37, %cl4801; X86-AVX512-NEXT: kmovd %ecx, %k14802; X86-AVX512-NEXT: vmovdqu64 %zmm2, (%eax) {%k1}4803; X86-AVX512-NEXT: vmovups %zmm3, 64(%eax)4804; X86-AVX512-NEXT: vzeroupper4805; X86-AVX512-NEXT: retl4806 %mask = icmp eq <16 x i64> %trigger, zeroinitializer4807 call void @llvm.masked.store.v16i64.p0(<16 x i64> %val, ptr %addr, i32 4, <16 x i1><i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)4808 ret void4809}4810 4811; When only one element of the mask is set, reduce to a scalar store.4812 4813define void @one_mask_bit_set1(ptr %addr, <4 x i32> %val) nounwind {4814; SSE-LABEL: one_mask_bit_set1:4815; SSE: ## %bb.0:4816; SSE-NEXT: movss %xmm0, (%rdi)4817; SSE-NEXT: retq4818;4819; AVX-LABEL: one_mask_bit_set1:4820; AVX: ## %bb.0:4821; AVX-NEXT: vmovss %xmm0, (%rdi)4822; AVX-NEXT: retq4823;4824; X86-AVX512-LABEL: one_mask_bit_set1:4825; X86-AVX512: ## %bb.0:4826; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax4827; X86-AVX512-NEXT: vmovss %xmm0, (%eax)4828; X86-AVX512-NEXT: retl4829 call void @llvm.masked.store.v4i32.p0(<4 x i32> %val, ptr %addr, i32 4, <4 x i1><i1 true, i1 false, i1 false, i1 false>)4830 ret void4831}4832 4833; Choose a different element to show that the correct address offset is produced.4834 4835define void @one_mask_bit_set2(ptr %addr, <4 x float> %val) nounwind {4836; SSE2-LABEL: one_mask_bit_set2:4837; SSE2: ## %bb.0:4838; SSE2-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1]4839; SSE2-NEXT: movss %xmm0, 8(%rdi)4840; SSE2-NEXT: retq4841;4842; SSE4-LABEL: one_mask_bit_set2:4843; SSE4: ## %bb.0:4844; SSE4-NEXT: extractps $2, %xmm0, 8(%rdi)4845; SSE4-NEXT: retq4846;4847; AVX-LABEL: one_mask_bit_set2:4848; AVX: ## %bb.0:4849; AVX-NEXT: vextractps $2, %xmm0, 8(%rdi)4850; AVX-NEXT: retq4851;4852; X86-AVX512-LABEL: one_mask_bit_set2:4853; X86-AVX512: ## %bb.0:4854; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax4855; X86-AVX512-NEXT: vextractps $2, %xmm0, 8(%eax)4856; X86-AVX512-NEXT: retl4857 call void @llvm.masked.store.v4f32.p0(<4 x float> %val, ptr %addr, i32 4, <4 x i1><i1 false, i1 false, i1 true, i1 false>)4858 ret void4859}4860 4861; Choose a different scalar type and a high element of a 256-bit vector because AVX doesn't support those evenly.4862 4863define void @one_mask_bit_set3(ptr %addr, <4 x i64> %val) nounwind {4864; SSE-LABEL: one_mask_bit_set3:4865; SSE: ## %bb.0:4866; SSE-NEXT: movlps %xmm1, 16(%rdi)4867; SSE-NEXT: retq4868;4869; AVX-LABEL: one_mask_bit_set3:4870; AVX: ## %bb.0:4871; AVX-NEXT: vextractf128 $1, %ymm0, %xmm04872; AVX-NEXT: vmovlps %xmm0, 16(%rdi)4873; AVX-NEXT: vzeroupper4874; AVX-NEXT: retq4875;4876; X86-AVX512-LABEL: one_mask_bit_set3:4877; X86-AVX512: ## %bb.0:4878; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax4879; X86-AVX512-NEXT: vextractf128 $1, %ymm0, %xmm04880; X86-AVX512-NEXT: vmovlps %xmm0, 16(%eax)4881; X86-AVX512-NEXT: vzeroupper4882; X86-AVX512-NEXT: retl4883 call void @llvm.masked.store.v4i64.p0(<4 x i64> %val, ptr %addr, i32 4, <4 x i1><i1 false, i1 false, i1 true, i1 false>)4884 ret void4885}4886 4887; Choose a different scalar type and a high element of a 256-bit vector because AVX doesn't support those evenly.4888 4889define void @one_mask_bit_set4(ptr %addr, <4 x double> %val) nounwind {4890; SSE-LABEL: one_mask_bit_set4:4891; SSE: ## %bb.0:4892; SSE-NEXT: movhps %xmm1, 24(%rdi)4893; SSE-NEXT: retq4894;4895; AVX-LABEL: one_mask_bit_set4:4896; AVX: ## %bb.0:4897; AVX-NEXT: vextractf128 $1, %ymm0, %xmm04898; AVX-NEXT: vmovhps %xmm0, 24(%rdi)4899; AVX-NEXT: vzeroupper4900; AVX-NEXT: retq4901;4902; X86-AVX512-LABEL: one_mask_bit_set4:4903; X86-AVX512: ## %bb.0:4904; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax4905; X86-AVX512-NEXT: vextractf128 $1, %ymm0, %xmm04906; X86-AVX512-NEXT: vmovhps %xmm0, 24(%eax)4907; X86-AVX512-NEXT: vzeroupper4908; X86-AVX512-NEXT: retl4909 call void @llvm.masked.store.v4f64.p0(<4 x double> %val, ptr %addr, i32 4, <4 x i1><i1 false, i1 false, i1 false, i1 true>)4910 ret void4911}4912 4913; Try a 512-bit vector to make sure AVX doesn't die and AVX512 works as expected.4914 4915define void @one_mask_bit_set5(ptr %addr, <8 x double> %val) nounwind {4916; SSE-LABEL: one_mask_bit_set5:4917; SSE: ## %bb.0:4918; SSE-NEXT: movlps %xmm3, 48(%rdi)4919; SSE-NEXT: retq4920;4921; AVX1OR2-LABEL: one_mask_bit_set5:4922; AVX1OR2: ## %bb.0:4923; AVX1OR2-NEXT: vextractf128 $1, %ymm1, %xmm04924; AVX1OR2-NEXT: vmovlps %xmm0, 48(%rdi)4925; AVX1OR2-NEXT: vzeroupper4926; AVX1OR2-NEXT: retq4927;4928; AVX512-LABEL: one_mask_bit_set5:4929; AVX512: ## %bb.0:4930; AVX512-NEXT: vextractf32x4 $3, %zmm0, %xmm04931; AVX512-NEXT: vmovlps %xmm0, 48(%rdi)4932; AVX512-NEXT: vzeroupper4933; AVX512-NEXT: retq4934;4935; X86-AVX512-LABEL: one_mask_bit_set5:4936; X86-AVX512: ## %bb.0:4937; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax4938; X86-AVX512-NEXT: vextractf32x4 $3, %zmm0, %xmm04939; X86-AVX512-NEXT: vmovlps %xmm0, 48(%eax)4940; X86-AVX512-NEXT: vzeroupper4941; X86-AVX512-NEXT: retl4942 call void @llvm.masked.store.v8f64.p0(<8 x double> %val, ptr %addr, i32 4, <8 x i1><i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false>)4943 ret void4944}4945 4946; Try one elt in each half of a vector that needs to split4947define void @one_mask_bit_set6(ptr %addr, <16 x i64> %val) nounwind {4948; SSE2-LABEL: one_mask_bit_set6:4949; SSE2: ## %bb.0:4950; SSE2-NEXT: movlps %xmm3, 48(%rdi)4951; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]4952; SSE2-NEXT: movq %xmm0, 88(%rdi)4953; SSE2-NEXT: retq4954;4955; SSE4-LABEL: one_mask_bit_set6:4956; SSE4: ## %bb.0:4957; SSE4-NEXT: movlps %xmm3, 48(%rdi)4958; SSE4-NEXT: pextrq $1, %xmm5, 88(%rdi)4959; SSE4-NEXT: retq4960;4961; AVX1-LABEL: one_mask_bit_set6:4962; AVX1: ## %bb.0:4963; AVX1-NEXT: vmovapd {{.*#+}} ymm0 = [0,0,0,18446744073709551615]4964; AVX1-NEXT: vmaskmovpd %ymm2, %ymm0, 64(%rdi)4965; AVX1-NEXT: vmovapd {{.*#+}} ymm0 = [0,0,18446744073709551615,0]4966; AVX1-NEXT: vmaskmovpd %ymm1, %ymm0, 32(%rdi)4967; AVX1-NEXT: vzeroupper4968; AVX1-NEXT: retq4969;4970; AVX2-LABEL: one_mask_bit_set6:4971; AVX2: ## %bb.0:4972; AVX2-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,0,0,18446744073709551615]4973; AVX2-NEXT: vpmaskmovq %ymm2, %ymm0, 64(%rdi)4974; AVX2-NEXT: vpmovsxbq {{.*#+}} ymm0 = [0,0,18446744073709551615,0]4975; AVX2-NEXT: vpmaskmovq %ymm1, %ymm0, 32(%rdi)4976; AVX2-NEXT: vzeroupper4977; AVX2-NEXT: retq4978;4979; AVX512-LABEL: one_mask_bit_set6:4980; AVX512: ## %bb.0:4981; AVX512-NEXT: vextractf32x4 $3, %zmm0, %xmm04982; AVX512-NEXT: vmovlps %xmm0, 48(%rdi)4983; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm04984; AVX512-NEXT: vpextrq $1, %xmm0, 88(%rdi)4985; AVX512-NEXT: vzeroupper4986; AVX512-NEXT: retq4987;4988; X86-AVX512-LABEL: one_mask_bit_set6:4989; X86-AVX512: ## %bb.0:4990; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax4991; X86-AVX512-NEXT: vextractf32x4 $3, %zmm0, %xmm04992; X86-AVX512-NEXT: vmovlps %xmm0, 48(%eax)4993; X86-AVX512-NEXT: vextractf128 $1, %ymm1, %xmm04994; X86-AVX512-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,3,0,1]4995; X86-AVX512-NEXT: vmovlps %xmm0, 88(%eax)4996; X86-AVX512-NEXT: vzeroupper4997; X86-AVX512-NEXT: retl4998 call void @llvm.masked.store.v16i64.p0(<16 x i64> %val, ptr %addr, i32 4, <16 x i1><i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false>)4999 ret void5000}5001 5002define void @top_bits_unset_stack() nounwind {5003; SSE-LABEL: top_bits_unset_stack:5004; SSE: ## %bb.0: ## %entry5005; SSE-NEXT: xorps %xmm0, %xmm05006; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)5007; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)5008; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)5009; SSE-NEXT: retq5010;5011; AVX1OR2-LABEL: top_bits_unset_stack:5012; AVX1OR2: ## %bb.0: ## %entry5013; AVX1OR2-NEXT: vxorpd %xmm0, %xmm0, %xmm05014; AVX1OR2-NEXT: vmovapd {{.*#+}} ymm1 = [18446744073709551615,18446744073709551615,0,0]5015; AVX1OR2-NEXT: vmaskmovpd %ymm0, %ymm1, -{{[0-9]+}}(%rsp)5016; AVX1OR2-NEXT: vmovupd %ymm0, -{{[0-9]+}}(%rsp)5017; AVX1OR2-NEXT: vzeroupper5018; AVX1OR2-NEXT: retq5019;5020; AVX512F-LABEL: top_bits_unset_stack:5021; AVX512F: ## %bb.0: ## %entry5022; AVX512F-NEXT: vxorpd %xmm0, %xmm0, %xmm05023; AVX512F-NEXT: movb $63, %al5024; AVX512F-NEXT: kmovw %eax, %k15025; AVX512F-NEXT: vmovupd %zmm0, -{{[0-9]+}}(%rsp) {%k1}5026; AVX512F-NEXT: vzeroupper5027; AVX512F-NEXT: retq5028;5029; AVX512VLDQ-LABEL: top_bits_unset_stack:5030; AVX512VLDQ: ## %bb.0: ## %entry5031; AVX512VLDQ-NEXT: vxorpd %xmm0, %xmm0, %xmm05032; AVX512VLDQ-NEXT: movb $63, %al5033; AVX512VLDQ-NEXT: kmovw %eax, %k15034; AVX512VLDQ-NEXT: vmovupd %zmm0, -{{[0-9]+}}(%rsp) {%k1}5035; AVX512VLDQ-NEXT: vzeroupper5036; AVX512VLDQ-NEXT: retq5037;5038; AVX512VLBW-LABEL: top_bits_unset_stack:5039; AVX512VLBW: ## %bb.0: ## %entry5040; AVX512VLBW-NEXT: vxorpd %xmm0, %xmm0, %xmm05041; AVX512VLBW-NEXT: movb $63, %al5042; AVX512VLBW-NEXT: kmovd %eax, %k15043; AVX512VLBW-NEXT: vmovupd %zmm0, -{{[0-9]+}}(%rsp) {%k1}5044; AVX512VLBW-NEXT: vzeroupper5045; AVX512VLBW-NEXT: retq5046;5047; X86-AVX512-LABEL: top_bits_unset_stack:5048; X86-AVX512: ## %bb.0: ## %entry5049; X86-AVX512-NEXT: subl $76, %esp5050; X86-AVX512-NEXT: vxorpd %xmm0, %xmm0, %xmm05051; X86-AVX512-NEXT: movb $63, %al5052; X86-AVX512-NEXT: kmovd %eax, %k15053; X86-AVX512-NEXT: vmovupd %zmm0, (%esp) {%k1}5054; X86-AVX512-NEXT: addl $76, %esp5055; X86-AVX512-NEXT: vzeroupper5056; X86-AVX512-NEXT: retl5057entry:5058 %P.i150.i.i = alloca [3 x [3 x double]], align 165059 call void @llvm.masked.store.v8f64.p0(<8 x double> zeroinitializer, ptr %P.i150.i.i, i32 8, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false>)5060 ret void5061}5062 5063 5064; SimplifyDemandedBits eliminates an ashr here.5065 5066define void @masked_store_bool_mask_demand_trunc_sext(<4 x double> %x, ptr %p, <4 x i32> %masksrc) nounwind {5067; SSE-LABEL: masked_store_bool_mask_demand_trunc_sext:5068; SSE: ## %bb.0:5069; SSE-NEXT: pslld $31, %xmm25070; SSE-NEXT: movmskps %xmm2, %eax5071; SSE-NEXT: testb $1, %al5072; SSE-NEXT: jne LBB26_15073; SSE-NEXT: ## %bb.2: ## %else5074; SSE-NEXT: testb $2, %al5075; SSE-NEXT: jne LBB26_35076; SSE-NEXT: LBB26_4: ## %else25077; SSE-NEXT: testb $4, %al5078; SSE-NEXT: jne LBB26_55079; SSE-NEXT: LBB26_6: ## %else45080; SSE-NEXT: testb $8, %al5081; SSE-NEXT: jne LBB26_75082; SSE-NEXT: LBB26_8: ## %else65083; SSE-NEXT: retq5084; SSE-NEXT: LBB26_1: ## %cond.store5085; SSE-NEXT: movlps %xmm0, (%rdi)5086; SSE-NEXT: testb $2, %al5087; SSE-NEXT: je LBB26_45088; SSE-NEXT: LBB26_3: ## %cond.store15089; SSE-NEXT: movhps %xmm0, 8(%rdi)5090; SSE-NEXT: testb $4, %al5091; SSE-NEXT: je LBB26_65092; SSE-NEXT: LBB26_5: ## %cond.store35093; SSE-NEXT: movlps %xmm1, 16(%rdi)5094; SSE-NEXT: testb $8, %al5095; SSE-NEXT: je LBB26_85096; SSE-NEXT: LBB26_7: ## %cond.store55097; SSE-NEXT: movhps %xmm1, 24(%rdi)5098; SSE-NEXT: retq5099;5100; AVX1-LABEL: masked_store_bool_mask_demand_trunc_sext:5101; AVX1: ## %bb.0:5102; AVX1-NEXT: vpslld $31, %xmm1, %xmm15103; AVX1-NEXT: vpmovsxdq %xmm1, %xmm25104; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]5105; AVX1-NEXT: vpmovsxdq %xmm1, %xmm15106; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm15107; AVX1-NEXT: vmaskmovpd %ymm0, %ymm1, (%rdi)5108; AVX1-NEXT: vzeroupper5109; AVX1-NEXT: retq5110;5111; AVX2-LABEL: masked_store_bool_mask_demand_trunc_sext:5112; AVX2: ## %bb.0:5113; AVX2-NEXT: vpslld $31, %xmm1, %xmm15114; AVX2-NEXT: vpmovsxdq %xmm1, %ymm15115; AVX2-NEXT: vmaskmovpd %ymm0, %ymm1, (%rdi)5116; AVX2-NEXT: vzeroupper5117; AVX2-NEXT: retq5118;5119; AVX512F-LABEL: masked_store_bool_mask_demand_trunc_sext:5120; AVX512F: ## %bb.0:5121; AVX512F-NEXT: ## kill: def $ymm0 killed $ymm0 def $zmm05122; AVX512F-NEXT: vpslld $31, %xmm1, %xmm15123; AVX512F-NEXT: vptestmd %zmm1, %zmm1, %k05124; AVX512F-NEXT: kshiftlw $12, %k0, %k05125; AVX512F-NEXT: kshiftrw $12, %k0, %k15126; AVX512F-NEXT: vmovupd %zmm0, (%rdi) {%k1}5127; AVX512F-NEXT: vzeroupper5128; AVX512F-NEXT: retq5129;5130; AVX512VLDQ-LABEL: masked_store_bool_mask_demand_trunc_sext:5131; AVX512VLDQ: ## %bb.0:5132; AVX512VLDQ-NEXT: vpslld $31, %xmm1, %xmm15133; AVX512VLDQ-NEXT: vpmovd2m %xmm1, %k15134; AVX512VLDQ-NEXT: vmovupd %ymm0, (%rdi) {%k1}5135; AVX512VLDQ-NEXT: vzeroupper5136; AVX512VLDQ-NEXT: retq5137;5138; AVX512VLBW-LABEL: masked_store_bool_mask_demand_trunc_sext:5139; AVX512VLBW: ## %bb.0:5140; AVX512VLBW-NEXT: vpslld $31, %xmm1, %xmm15141; AVX512VLBW-NEXT: vptestmd %xmm1, %xmm1, %k15142; AVX512VLBW-NEXT: vmovupd %ymm0, (%rdi) {%k1}5143; AVX512VLBW-NEXT: vzeroupper5144; AVX512VLBW-NEXT: retq5145;5146; X86-AVX512-LABEL: masked_store_bool_mask_demand_trunc_sext:5147; X86-AVX512: ## %bb.0:5148; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax5149; X86-AVX512-NEXT: vpslld $31, %xmm1, %xmm15150; X86-AVX512-NEXT: vpmovd2m %xmm1, %k15151; X86-AVX512-NEXT: vmovupd %ymm0, (%eax) {%k1}5152; X86-AVX512-NEXT: vzeroupper5153; X86-AVX512-NEXT: retl5154 %sext = sext <4 x i32> %masksrc to <4 x i64>5155 %boolmask = trunc <4 x i64> %sext to <4 x i1>5156 call void @llvm.masked.store.v4f64.p0(<4 x double> %x, ptr %p, i32 4, <4 x i1> %boolmask)5157 ret void5158}5159 5160; PR266975161 5162define void @one_mask_bit_set1_variable(ptr %addr, <4 x float> %val, <4 x i32> %mask) nounwind {5163; SSE2-LABEL: one_mask_bit_set1_variable:5164; SSE2: ## %bb.0:5165; SSE2-NEXT: movmskps %xmm1, %eax5166; SSE2-NEXT: testb $1, %al5167; SSE2-NEXT: jne LBB27_15168; SSE2-NEXT: ## %bb.2: ## %else5169; SSE2-NEXT: testb $2, %al5170; SSE2-NEXT: jne LBB27_35171; SSE2-NEXT: LBB27_4: ## %else25172; SSE2-NEXT: testb $4, %al5173; SSE2-NEXT: jne LBB27_55174; SSE2-NEXT: LBB27_6: ## %else45175; SSE2-NEXT: testb $8, %al5176; SSE2-NEXT: jne LBB27_75177; SSE2-NEXT: LBB27_8: ## %else65178; SSE2-NEXT: retq5179; SSE2-NEXT: LBB27_1: ## %cond.store5180; SSE2-NEXT: movss %xmm0, (%rdi)5181; SSE2-NEXT: testb $2, %al5182; SSE2-NEXT: je LBB27_45183; SSE2-NEXT: LBB27_3: ## %cond.store15184; SSE2-NEXT: movaps %xmm0, %xmm15185; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]5186; SSE2-NEXT: movss %xmm1, 4(%rdi)5187; SSE2-NEXT: testb $4, %al5188; SSE2-NEXT: je LBB27_65189; SSE2-NEXT: LBB27_5: ## %cond.store35190; SSE2-NEXT: movaps %xmm0, %xmm15191; SSE2-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]5192; SSE2-NEXT: movss %xmm1, 8(%rdi)5193; SSE2-NEXT: testb $8, %al5194; SSE2-NEXT: je LBB27_85195; SSE2-NEXT: LBB27_7: ## %cond.store55196; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]5197; SSE2-NEXT: movss %xmm0, 12(%rdi)5198; SSE2-NEXT: retq5199;5200; SSE4-LABEL: one_mask_bit_set1_variable:5201; SSE4: ## %bb.0:5202; SSE4-NEXT: movmskps %xmm1, %eax5203; SSE4-NEXT: testb $1, %al5204; SSE4-NEXT: jne LBB27_15205; SSE4-NEXT: ## %bb.2: ## %else5206; SSE4-NEXT: testb $2, %al5207; SSE4-NEXT: jne LBB27_35208; SSE4-NEXT: LBB27_4: ## %else25209; SSE4-NEXT: testb $4, %al5210; SSE4-NEXT: jne LBB27_55211; SSE4-NEXT: LBB27_6: ## %else45212; SSE4-NEXT: testb $8, %al5213; SSE4-NEXT: jne LBB27_75214; SSE4-NEXT: LBB27_8: ## %else65215; SSE4-NEXT: retq5216; SSE4-NEXT: LBB27_1: ## %cond.store5217; SSE4-NEXT: movss %xmm0, (%rdi)5218; SSE4-NEXT: testb $2, %al5219; SSE4-NEXT: je LBB27_45220; SSE4-NEXT: LBB27_3: ## %cond.store15221; SSE4-NEXT: extractps $1, %xmm0, 4(%rdi)5222; SSE4-NEXT: testb $4, %al5223; SSE4-NEXT: je LBB27_65224; SSE4-NEXT: LBB27_5: ## %cond.store35225; SSE4-NEXT: extractps $2, %xmm0, 8(%rdi)5226; SSE4-NEXT: testb $8, %al5227; SSE4-NEXT: je LBB27_85228; SSE4-NEXT: LBB27_7: ## %cond.store55229; SSE4-NEXT: extractps $3, %xmm0, 12(%rdi)5230; SSE4-NEXT: retq5231;5232; AVX1OR2-LABEL: one_mask_bit_set1_variable:5233; AVX1OR2: ## %bb.0:5234; AVX1OR2-NEXT: vmaskmovps %xmm0, %xmm1, (%rdi)5235; AVX1OR2-NEXT: retq5236;5237; AVX512F-LABEL: one_mask_bit_set1_variable:5238; AVX512F: ## %bb.0:5239; AVX512F-NEXT: ## kill: def $xmm1 killed $xmm1 def $zmm15240; AVX512F-NEXT: ## kill: def $xmm0 killed $xmm0 def $zmm05241; AVX512F-NEXT: vptestmd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %k05242; AVX512F-NEXT: kshiftlw $12, %k0, %k05243; AVX512F-NEXT: kshiftrw $12, %k0, %k15244; AVX512F-NEXT: vmovups %zmm0, (%rdi) {%k1}5245; AVX512F-NEXT: vzeroupper5246; AVX512F-NEXT: retq5247;5248; AVX512VL-LABEL: one_mask_bit_set1_variable:5249; AVX512VL: ## %bb.0:5250; AVX512VL-NEXT: vptestmd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %k15251; AVX512VL-NEXT: vmovups %xmm0, (%rdi) {%k1}5252; AVX512VL-NEXT: retq5253;5254; X86-AVX512-LABEL: one_mask_bit_set1_variable:5255; X86-AVX512: ## %bb.0:5256; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax5257; X86-AVX512-NEXT: vptestmd {{\.?LCPI[0-9]+_[0-9]+}}{1to4}, %xmm1, %k15258; X86-AVX512-NEXT: vmovups %xmm0, (%eax) {%k1}5259; X86-AVX512-NEXT: retl5260 %mask_signbit = and <4 x i32> %mask, <i32 2147483648, i32 2147483648, i32 2147483648, i32 2147483648>5261 %mask_bool = icmp ne <4 x i32> %mask_signbit, zeroinitializer5262 call void @llvm.masked.store.v4f32.p0(<4 x float> %val, ptr %addr, i32 1, <4 x i1> %mask_bool)5263 ret void5264}5265 5266; This needs to be widened to v4i32.5267; This used to assert in type legalization. PR384365268; FIXME: The codegen for AVX512 should use KSHIFT to zero the upper bits of the mask.5269define void @widen_masked_store(<3 x i32> %v, ptr %p, <3 x i1> %mask) nounwind {5270; SSE2-LABEL: widen_masked_store:5271; SSE2: ## %bb.0:5272; SSE2-NEXT: andb $1, %sil5273; SSE2-NEXT: andb $1, %dl5274; SSE2-NEXT: addb %dl, %dl5275; SSE2-NEXT: orb %sil, %dl5276; SSE2-NEXT: andb $1, %cl5277; SSE2-NEXT: shlb $2, %cl5278; SSE2-NEXT: orb %dl, %cl5279; SSE2-NEXT: testb $1, %cl5280; SSE2-NEXT: jne LBB28_15281; SSE2-NEXT: ## %bb.2: ## %else5282; SSE2-NEXT: testb $2, %cl5283; SSE2-NEXT: jne LBB28_35284; SSE2-NEXT: LBB28_4: ## %else25285; SSE2-NEXT: testb $4, %cl5286; SSE2-NEXT: jne LBB28_55287; SSE2-NEXT: LBB28_6: ## %else45288; SSE2-NEXT: retq5289; SSE2-NEXT: LBB28_1: ## %cond.store5290; SSE2-NEXT: movd %xmm0, (%rdi)5291; SSE2-NEXT: testb $2, %cl5292; SSE2-NEXT: je LBB28_45293; SSE2-NEXT: LBB28_3: ## %cond.store15294; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]5295; SSE2-NEXT: movd %xmm1, 4(%rdi)5296; SSE2-NEXT: testb $4, %cl5297; SSE2-NEXT: je LBB28_65298; SSE2-NEXT: LBB28_5: ## %cond.store35299; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]5300; SSE2-NEXT: movd %xmm0, 8(%rdi)5301; SSE2-NEXT: retq5302;5303; SSE4-LABEL: widen_masked_store:5304; SSE4: ## %bb.0:5305; SSE4-NEXT: andb $1, %sil5306; SSE4-NEXT: andb $1, %dl5307; SSE4-NEXT: addb %dl, %dl5308; SSE4-NEXT: orb %sil, %dl5309; SSE4-NEXT: andb $1, %cl5310; SSE4-NEXT: shlb $2, %cl5311; SSE4-NEXT: orb %dl, %cl5312; SSE4-NEXT: testb $1, %cl5313; SSE4-NEXT: jne LBB28_15314; SSE4-NEXT: ## %bb.2: ## %else5315; SSE4-NEXT: testb $2, %cl5316; SSE4-NEXT: jne LBB28_35317; SSE4-NEXT: LBB28_4: ## %else25318; SSE4-NEXT: testb $4, %cl5319; SSE4-NEXT: jne LBB28_55320; SSE4-NEXT: LBB28_6: ## %else45321; SSE4-NEXT: retq5322; SSE4-NEXT: LBB28_1: ## %cond.store5323; SSE4-NEXT: movss %xmm0, (%rdi)5324; SSE4-NEXT: testb $2, %cl5325; SSE4-NEXT: je LBB28_45326; SSE4-NEXT: LBB28_3: ## %cond.store15327; SSE4-NEXT: extractps $1, %xmm0, 4(%rdi)5328; SSE4-NEXT: testb $4, %cl5329; SSE4-NEXT: je LBB28_65330; SSE4-NEXT: LBB28_5: ## %cond.store35331; SSE4-NEXT: extractps $2, %xmm0, 8(%rdi)5332; SSE4-NEXT: retq5333;5334; AVX1-LABEL: widen_masked_store:5335; AVX1: ## %bb.0:5336; AVX1-NEXT: vmovd %edx, %xmm15337; AVX1-NEXT: vmovd %esi, %xmm25338; AVX1-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]5339; AVX1-NEXT: vmovd %ecx, %xmm25340; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]5341; AVX1-NEXT: vpslld $31, %xmm1, %xmm15342; AVX1-NEXT: vmaskmovps %xmm0, %xmm1, (%rdi)5343; AVX1-NEXT: retq5344;5345; AVX2-LABEL: widen_masked_store:5346; AVX2: ## %bb.0:5347; AVX2-NEXT: vmovd %edx, %xmm15348; AVX2-NEXT: vmovd %esi, %xmm25349; AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]5350; AVX2-NEXT: vmovd %ecx, %xmm25351; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]5352; AVX2-NEXT: vpslld $31, %xmm1, %xmm15353; AVX2-NEXT: vpmaskmovd %xmm0, %xmm1, (%rdi)5354; AVX2-NEXT: retq5355;5356; AVX512F-LABEL: widen_masked_store:5357; AVX512F: ## %bb.0:5358; AVX512F-NEXT: ## kill: def $xmm0 killed $xmm0 def $zmm05359; AVX512F-NEXT: andl $1, %esi5360; AVX512F-NEXT: kmovw %esi, %k05361; AVX512F-NEXT: kmovw %edx, %k15362; AVX512F-NEXT: kshiftlw $15, %k1, %k15363; AVX512F-NEXT: kshiftrw $14, %k1, %k15364; AVX512F-NEXT: korw %k1, %k0, %k05365; AVX512F-NEXT: movw $-5, %ax5366; AVX512F-NEXT: kmovw %eax, %k15367; AVX512F-NEXT: kandw %k1, %k0, %k05368; AVX512F-NEXT: kmovw %ecx, %k15369; AVX512F-NEXT: kshiftlw $15, %k1, %k15370; AVX512F-NEXT: kshiftrw $13, %k1, %k15371; AVX512F-NEXT: korw %k1, %k0, %k05372; AVX512F-NEXT: movb $7, %al5373; AVX512F-NEXT: kmovw %eax, %k15374; AVX512F-NEXT: kandw %k1, %k0, %k05375; AVX512F-NEXT: kshiftlw $12, %k0, %k05376; AVX512F-NEXT: kshiftrw $12, %k0, %k15377; AVX512F-NEXT: vmovdqu32 %zmm0, (%rdi) {%k1}5378; AVX512F-NEXT: vzeroupper5379; AVX512F-NEXT: retq5380;5381; AVX512VLDQ-LABEL: widen_masked_store:5382; AVX512VLDQ: ## %bb.0:5383; AVX512VLDQ-NEXT: kmovw %edx, %k05384; AVX512VLDQ-NEXT: kshiftlb $7, %k0, %k05385; AVX512VLDQ-NEXT: kshiftrb $6, %k0, %k05386; AVX512VLDQ-NEXT: kmovw %esi, %k15387; AVX512VLDQ-NEXT: kshiftlb $7, %k1, %k15388; AVX512VLDQ-NEXT: kshiftrb $7, %k1, %k15389; AVX512VLDQ-NEXT: korw %k0, %k1, %k05390; AVX512VLDQ-NEXT: movb $-5, %al5391; AVX512VLDQ-NEXT: kmovw %eax, %k15392; AVX512VLDQ-NEXT: kandw %k1, %k0, %k05393; AVX512VLDQ-NEXT: kmovw %ecx, %k15394; AVX512VLDQ-NEXT: kshiftlb $7, %k1, %k15395; AVX512VLDQ-NEXT: kshiftrb $5, %k1, %k15396; AVX512VLDQ-NEXT: korw %k1, %k0, %k05397; AVX512VLDQ-NEXT: movb $7, %al5398; AVX512VLDQ-NEXT: kmovw %eax, %k15399; AVX512VLDQ-NEXT: kandw %k1, %k0, %k15400; AVX512VLDQ-NEXT: vmovdqa32 %xmm0, (%rdi) {%k1}5401; AVX512VLDQ-NEXT: retq5402;5403; AVX512VLBW-LABEL: widen_masked_store:5404; AVX512VLBW: ## %bb.0:5405; AVX512VLBW-NEXT: andl $1, %esi5406; AVX512VLBW-NEXT: kmovw %esi, %k05407; AVX512VLBW-NEXT: kmovd %edx, %k15408; AVX512VLBW-NEXT: kshiftlw $15, %k1, %k15409; AVX512VLBW-NEXT: kshiftrw $14, %k1, %k15410; AVX512VLBW-NEXT: korw %k1, %k0, %k05411; AVX512VLBW-NEXT: movw $-5, %ax5412; AVX512VLBW-NEXT: kmovd %eax, %k15413; AVX512VLBW-NEXT: kandw %k1, %k0, %k05414; AVX512VLBW-NEXT: kmovd %ecx, %k15415; AVX512VLBW-NEXT: kshiftlw $15, %k1, %k15416; AVX512VLBW-NEXT: kshiftrw $13, %k1, %k15417; AVX512VLBW-NEXT: korw %k1, %k0, %k05418; AVX512VLBW-NEXT: movb $7, %al5419; AVX512VLBW-NEXT: kmovd %eax, %k15420; AVX512VLBW-NEXT: kandw %k1, %k0, %k15421; AVX512VLBW-NEXT: vmovdqa32 %xmm0, (%rdi) {%k1}5422; AVX512VLBW-NEXT: retq5423;5424; X86-AVX512-LABEL: widen_masked_store:5425; X86-AVX512: ## %bb.0:5426; X86-AVX512-NEXT: kmovb {{[0-9]+}}(%esp), %k05427; X86-AVX512-NEXT: kshiftlb $7, %k0, %k05428; X86-AVX512-NEXT: kshiftrb $6, %k0, %k05429; X86-AVX512-NEXT: kmovb {{[0-9]+}}(%esp), %k15430; X86-AVX512-NEXT: kshiftlb $7, %k1, %k15431; X86-AVX512-NEXT: kshiftrb $7, %k1, %k15432; X86-AVX512-NEXT: korw %k0, %k1, %k05433; X86-AVX512-NEXT: movb $-5, %al5434; X86-AVX512-NEXT: kmovd %eax, %k15435; X86-AVX512-NEXT: kandw %k1, %k0, %k05436; X86-AVX512-NEXT: kmovb {{[0-9]+}}(%esp), %k15437; X86-AVX512-NEXT: kshiftlb $7, %k1, %k15438; X86-AVX512-NEXT: kshiftrb $5, %k1, %k15439; X86-AVX512-NEXT: korw %k1, %k0, %k05440; X86-AVX512-NEXT: movb $7, %al5441; X86-AVX512-NEXT: kmovd %eax, %k15442; X86-AVX512-NEXT: kandw %k1, %k0, %k15443; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax5444; X86-AVX512-NEXT: vmovdqa32 %xmm0, (%eax) {%k1}5445; X86-AVX512-NEXT: retl5446 call void @llvm.masked.store.v3i32.p0(<3 x i32> %v, ptr %p, i32 16, <3 x i1> %mask)5447 ret void5448}5449 5450define void @zero_mask(ptr %addr, <2 x double> %val) nounwind {5451; SSE-LABEL: zero_mask:5452; SSE: ## %bb.0:5453; SSE-NEXT: retq5454;5455; AVX-LABEL: zero_mask:5456; AVX: ## %bb.0:5457; AVX-NEXT: retq5458;5459; X86-AVX512-LABEL: zero_mask:5460; X86-AVX512: ## %bb.0:5461; X86-AVX512-NEXT: retl5462 call void @llvm.masked.store.v2f64.p0(<2 x double> %val, ptr %addr, i32 4, <2 x i1> zeroinitializer)5463 ret void5464}5465 5466define void @PR11210(<4 x float> %x, ptr %ptr, <4 x float> %y, <2 x i64> %mask) nounwind {5467; SSE2-LABEL: PR11210:5468; SSE2: ## %bb.0:5469; SSE2-NEXT: movmskps %xmm2, %eax5470; SSE2-NEXT: testb $1, %al5471; SSE2-NEXT: jne LBB30_15472; SSE2-NEXT: ## %bb.2: ## %else5473; SSE2-NEXT: testb $2, %al5474; SSE2-NEXT: jne LBB30_35475; SSE2-NEXT: LBB30_4: ## %else25476; SSE2-NEXT: testb $4, %al5477; SSE2-NEXT: jne LBB30_55478; SSE2-NEXT: LBB30_6: ## %else45479; SSE2-NEXT: testb $8, %al5480; SSE2-NEXT: jne LBB30_75481; SSE2-NEXT: LBB30_8: ## %else65482; SSE2-NEXT: testb $1, %al5483; SSE2-NEXT: jne LBB30_95484; SSE2-NEXT: LBB30_10: ## %else95485; SSE2-NEXT: testb $2, %al5486; SSE2-NEXT: jne LBB30_115487; SSE2-NEXT: LBB30_12: ## %else115488; SSE2-NEXT: testb $4, %al5489; SSE2-NEXT: jne LBB30_135490; SSE2-NEXT: LBB30_14: ## %else135491; SSE2-NEXT: testb $8, %al5492; SSE2-NEXT: jne LBB30_155493; SSE2-NEXT: LBB30_16: ## %else155494; SSE2-NEXT: retq5495; SSE2-NEXT: LBB30_1: ## %cond.store5496; SSE2-NEXT: movss %xmm0, (%rdi)5497; SSE2-NEXT: testb $2, %al5498; SSE2-NEXT: je LBB30_45499; SSE2-NEXT: LBB30_3: ## %cond.store15500; SSE2-NEXT: movaps %xmm0, %xmm25501; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[1,1]5502; SSE2-NEXT: movss %xmm2, 4(%rdi)5503; SSE2-NEXT: testb $4, %al5504; SSE2-NEXT: je LBB30_65505; SSE2-NEXT: LBB30_5: ## %cond.store35506; SSE2-NEXT: movaps %xmm0, %xmm25507; SSE2-NEXT: unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]5508; SSE2-NEXT: movss %xmm2, 8(%rdi)5509; SSE2-NEXT: testb $8, %al5510; SSE2-NEXT: je LBB30_85511; SSE2-NEXT: LBB30_7: ## %cond.store55512; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]5513; SSE2-NEXT: movss %xmm0, 12(%rdi)5514; SSE2-NEXT: testb $1, %al5515; SSE2-NEXT: je LBB30_105516; SSE2-NEXT: LBB30_9: ## %cond.store85517; SSE2-NEXT: movss %xmm1, (%rdi)5518; SSE2-NEXT: testb $2, %al5519; SSE2-NEXT: je LBB30_125520; SSE2-NEXT: LBB30_11: ## %cond.store105521; SSE2-NEXT: movaps %xmm1, %xmm05522; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1]5523; SSE2-NEXT: movss %xmm0, 4(%rdi)5524; SSE2-NEXT: testb $4, %al5525; SSE2-NEXT: je LBB30_145526; SSE2-NEXT: LBB30_13: ## %cond.store125527; SSE2-NEXT: movaps %xmm1, %xmm05528; SSE2-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]5529; SSE2-NEXT: movss %xmm0, 8(%rdi)5530; SSE2-NEXT: testb $8, %al5531; SSE2-NEXT: je LBB30_165532; SSE2-NEXT: LBB30_15: ## %cond.store145533; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]5534; SSE2-NEXT: movss %xmm1, 12(%rdi)5535; SSE2-NEXT: retq5536;5537; SSE4-LABEL: PR11210:5538; SSE4: ## %bb.0:5539; SSE4-NEXT: movmskps %xmm2, %eax5540; SSE4-NEXT: testb $1, %al5541; SSE4-NEXT: jne LBB30_15542; SSE4-NEXT: ## %bb.2: ## %else5543; SSE4-NEXT: testb $2, %al5544; SSE4-NEXT: jne LBB30_35545; SSE4-NEXT: LBB30_4: ## %else25546; SSE4-NEXT: testb $4, %al5547; SSE4-NEXT: jne LBB30_55548; SSE4-NEXT: LBB30_6: ## %else45549; SSE4-NEXT: testb $8, %al5550; SSE4-NEXT: jne LBB30_75551; SSE4-NEXT: LBB30_8: ## %else65552; SSE4-NEXT: testb $1, %al5553; SSE4-NEXT: jne LBB30_95554; SSE4-NEXT: LBB30_10: ## %else95555; SSE4-NEXT: testb $2, %al5556; SSE4-NEXT: jne LBB30_115557; SSE4-NEXT: LBB30_12: ## %else115558; SSE4-NEXT: testb $4, %al5559; SSE4-NEXT: jne LBB30_135560; SSE4-NEXT: LBB30_14: ## %else135561; SSE4-NEXT: testb $8, %al5562; SSE4-NEXT: jne LBB30_155563; SSE4-NEXT: LBB30_16: ## %else155564; SSE4-NEXT: retq5565; SSE4-NEXT: LBB30_1: ## %cond.store5566; SSE4-NEXT: movss %xmm0, (%rdi)5567; SSE4-NEXT: testb $2, %al5568; SSE4-NEXT: je LBB30_45569; SSE4-NEXT: LBB30_3: ## %cond.store15570; SSE4-NEXT: extractps $1, %xmm0, 4(%rdi)5571; SSE4-NEXT: testb $4, %al5572; SSE4-NEXT: je LBB30_65573; SSE4-NEXT: LBB30_5: ## %cond.store35574; SSE4-NEXT: extractps $2, %xmm0, 8(%rdi)5575; SSE4-NEXT: testb $8, %al5576; SSE4-NEXT: je LBB30_85577; SSE4-NEXT: LBB30_7: ## %cond.store55578; SSE4-NEXT: extractps $3, %xmm0, 12(%rdi)5579; SSE4-NEXT: testb $1, %al5580; SSE4-NEXT: je LBB30_105581; SSE4-NEXT: LBB30_9: ## %cond.store85582; SSE4-NEXT: movss %xmm1, (%rdi)5583; SSE4-NEXT: testb $2, %al5584; SSE4-NEXT: je LBB30_125585; SSE4-NEXT: LBB30_11: ## %cond.store105586; SSE4-NEXT: extractps $1, %xmm1, 4(%rdi)5587; SSE4-NEXT: testb $4, %al5588; SSE4-NEXT: je LBB30_145589; SSE4-NEXT: LBB30_13: ## %cond.store125590; SSE4-NEXT: extractps $2, %xmm1, 8(%rdi)5591; SSE4-NEXT: testb $8, %al5592; SSE4-NEXT: je LBB30_165593; SSE4-NEXT: LBB30_15: ## %cond.store145594; SSE4-NEXT: extractps $3, %xmm1, 12(%rdi)5595; SSE4-NEXT: retq5596;5597; AVX1OR2-LABEL: PR11210:5598; AVX1OR2: ## %bb.0:5599; AVX1OR2-NEXT: vmaskmovps %xmm1, %xmm2, (%rdi)5600; AVX1OR2-NEXT: retq5601;5602; AVX512F-LABEL: PR11210:5603; AVX512F: ## %bb.0:5604; AVX512F-NEXT: ## kill: def $xmm2 killed $xmm2 def $zmm25605; AVX512F-NEXT: ## kill: def $xmm1 killed $xmm1 def $zmm15606; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm05607; AVX512F-NEXT: vpcmpgtd %zmm2, %zmm0, %k05608; AVX512F-NEXT: kshiftlw $12, %k0, %k05609; AVX512F-NEXT: kshiftrw $12, %k0, %k15610; AVX512F-NEXT: vmovups %zmm1, (%rdi) {%k1}5611; AVX512F-NEXT: vzeroupper5612; AVX512F-NEXT: retq5613;5614; AVX512VLDQ-LABEL: PR11210:5615; AVX512VLDQ: ## %bb.0:5616; AVX512VLDQ-NEXT: vpmovd2m %xmm2, %k15617; AVX512VLDQ-NEXT: vmovups %xmm1, (%rdi) {%k1}5618; AVX512VLDQ-NEXT: retq5619;5620; AVX512VLBW-LABEL: PR11210:5621; AVX512VLBW: ## %bb.0:5622; AVX512VLBW-NEXT: vpxor %xmm0, %xmm0, %xmm05623; AVX512VLBW-NEXT: vpcmpgtd %xmm2, %xmm0, %k15624; AVX512VLBW-NEXT: vmovups %xmm1, (%rdi) {%k1}5625; AVX512VLBW-NEXT: retq5626;5627; X86-AVX512-LABEL: PR11210:5628; X86-AVX512: ## %bb.0:5629; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax5630; X86-AVX512-NEXT: vpmovd2m %xmm2, %k15631; X86-AVX512-NEXT: vmovups %xmm1, (%eax) {%k1}5632; X86-AVX512-NEXT: retl5633 %bc = bitcast <2 x i64> %mask to <4 x i32>5634 %trunc = icmp slt <4 x i32> %bc, zeroinitializer5635 call void @llvm.masked.store.v4f32.p0(<4 x float> %x, ptr %ptr, i32 1, <4 x i1> %trunc)5636 call void @llvm.masked.store.v4f32.p0(<4 x float> %y, ptr %ptr, i32 1, <4 x i1> %trunc)5637 ret void5638}5639 5640define void @store_v24i32_v24i32_stride6_vf4_only_even_numbered_elts(ptr %trigger.ptr, ptr %val.ptr, ptr %dst) nounwind {5641; SSE-LABEL: store_v24i32_v24i32_stride6_vf4_only_even_numbered_elts:5642; SSE: ## %bb.0:5643; SSE-NEXT: pushq %rbp5644; SSE-NEXT: pushq %r155645; SSE-NEXT: pushq %r145646; SSE-NEXT: pushq %r135647; SSE-NEXT: pushq %r125648; SSE-NEXT: pushq %rbx5649; SSE-NEXT: movdqa (%rdi), %xmm15650; SSE-NEXT: movdqa 32(%rdi), %xmm25651; SSE-NEXT: movdqa 64(%rdi), %xmm05652; SSE-NEXT: movl 92(%rsi), %eax5653; SSE-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill5654; SSE-NEXT: movl 88(%rsi), %eax5655; SSE-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill5656; SSE-NEXT: movl 84(%rsi), %eax5657; SSE-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill5658; SSE-NEXT: movl 80(%rsi), %eax5659; SSE-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill5660; SSE-NEXT: movl 76(%rsi), %eax5661; SSE-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill5662; SSE-NEXT: movl 72(%rsi), %eax5663; SSE-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill5664; SSE-NEXT: movl 68(%rsi), %eax5665; SSE-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill5666; SSE-NEXT: movl 64(%rsi), %eax5667; SSE-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill5668; SSE-NEXT: movl 60(%rsi), %eax5669; SSE-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill5670; SSE-NEXT: movl 56(%rsi), %eax5671; SSE-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill5672; SSE-NEXT: movl 52(%rsi), %eax5673; SSE-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill5674; SSE-NEXT: packssdw 48(%rdi), %xmm25675; SSE-NEXT: packssdw 16(%rdi), %xmm15676; SSE-NEXT: packsswb %xmm2, %xmm15677; SSE-NEXT: packssdw 80(%rdi), %xmm05678; SSE-NEXT: packsswb %xmm0, %xmm05679; SSE-NEXT: pmovmskb %xmm1, %eax5680; SSE-NEXT: andl $21845, %eax ## imm = 0x55555681; SSE-NEXT: pmovmskb %xmm0, %edi5682; SSE-NEXT: andl $85, %edi5683; SSE-NEXT: shll $16, %edi5684; SSE-NEXT: orl %eax, %edi5685; SSE-NEXT: movl 48(%rsi), %r13d5686; SSE-NEXT: testb $1, %dil5687; SSE-NEXT: movl 44(%rsi), %eax5688; SSE-NEXT: movl 40(%rsi), %ecx5689; SSE-NEXT: movl 36(%rsi), %r8d5690; SSE-NEXT: movl 32(%rsi), %r9d5691; SSE-NEXT: movl 28(%rsi), %r10d5692; SSE-NEXT: movl 24(%rsi), %r11d5693; SSE-NEXT: movl 20(%rsi), %ebx5694; SSE-NEXT: movl 16(%rsi), %ebp5695; SSE-NEXT: movl 12(%rsi), %r14d5696; SSE-NEXT: movl 8(%rsi), %r15d5697; SSE-NEXT: movl 4(%rsi), %r12d5698; SSE-NEXT: jne LBB31_15699; SSE-NEXT: ## %bb.2: ## %else5700; SSE-NEXT: testb $2, %dil5701; SSE-NEXT: jne LBB31_35702; SSE-NEXT: LBB31_4: ## %else25703; SSE-NEXT: testb $4, %dil5704; SSE-NEXT: jne LBB31_55705; SSE-NEXT: LBB31_6: ## %else45706; SSE-NEXT: testb $8, %dil5707; SSE-NEXT: jne LBB31_75708; SSE-NEXT: LBB31_8: ## %else65709; SSE-NEXT: testb $16, %dil5710; SSE-NEXT: jne LBB31_95711; SSE-NEXT: LBB31_10: ## %else85712; SSE-NEXT: testb $32, %dil5713; SSE-NEXT: jne LBB31_115714; SSE-NEXT: LBB31_12: ## %else105715; SSE-NEXT: testb $64, %dil5716; SSE-NEXT: jne LBB31_135717; SSE-NEXT: LBB31_14: ## %else125718; SSE-NEXT: testb %dil, %dil5719; SSE-NEXT: js LBB31_155720; SSE-NEXT: LBB31_16: ## %else145721; SSE-NEXT: testl $256, %edi ## imm = 0x1005722; SSE-NEXT: jne LBB31_175723; SSE-NEXT: LBB31_18: ## %else165724; SSE-NEXT: testl $512, %edi ## imm = 0x2005725; SSE-NEXT: jne LBB31_195726; SSE-NEXT: LBB31_20: ## %else185727; SSE-NEXT: testl $1024, %edi ## imm = 0x4005728; SSE-NEXT: jne LBB31_215729; SSE-NEXT: LBB31_22: ## %else205730; SSE-NEXT: testl $2048, %edi ## imm = 0x8005731; SSE-NEXT: jne LBB31_235732; SSE-NEXT: LBB31_24: ## %else225733; SSE-NEXT: testl $4096, %edi ## imm = 0x10005734; SSE-NEXT: jne LBB31_255735; SSE-NEXT: LBB31_26: ## %else245736; SSE-NEXT: testl $8192, %edi ## imm = 0x20005737; SSE-NEXT: jne LBB31_275738; SSE-NEXT: LBB31_28: ## %else265739; SSE-NEXT: testl $16384, %edi ## imm = 0x40005740; SSE-NEXT: jne LBB31_295741; SSE-NEXT: LBB31_30: ## %else285742; SSE-NEXT: testw %di, %di5743; SSE-NEXT: js LBB31_315744; SSE-NEXT: LBB31_32: ## %else305745; SSE-NEXT: testl $65536, %edi ## imm = 0x100005746; SSE-NEXT: jne LBB31_335747; SSE-NEXT: LBB31_34: ## %else325748; SSE-NEXT: testl $131072, %edi ## imm = 0x200005749; SSE-NEXT: jne LBB31_355750; SSE-NEXT: LBB31_36: ## %else345751; SSE-NEXT: testl $262144, %edi ## imm = 0x400005752; SSE-NEXT: jne LBB31_375753; SSE-NEXT: LBB31_38: ## %else365754; SSE-NEXT: testl $524288, %edi ## imm = 0x800005755; SSE-NEXT: jne LBB31_395756; SSE-NEXT: LBB31_40: ## %else385757; SSE-NEXT: testl $1048576, %edi ## imm = 0x1000005758; SSE-NEXT: jne LBB31_415759; SSE-NEXT: LBB31_42: ## %else405760; SSE-NEXT: testl $2097152, %edi ## imm = 0x2000005761; SSE-NEXT: jne LBB31_435762; SSE-NEXT: LBB31_44: ## %else425763; SSE-NEXT: testl $4194304, %edi ## imm = 0x4000005764; SSE-NEXT: je LBB31_465765; SSE-NEXT: LBB31_45: ## %cond.store435766; SSE-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax ## 4-byte Reload5767; SSE-NEXT: movl %eax, 88(%rdx)5768; SSE-NEXT: LBB31_46: ## %else445769; SSE-NEXT: movb $1, %al5770; SSE-NEXT: testb %al, %al5771; SSE-NEXT: jne LBB31_485772; SSE-NEXT: ## %bb.47: ## %cond.store455773; SSE-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax ## 4-byte Reload5774; SSE-NEXT: movl %eax, 92(%rdx)5775; SSE-NEXT: LBB31_48: ## %else465776; SSE-NEXT: popq %rbx5777; SSE-NEXT: popq %r125778; SSE-NEXT: popq %r135779; SSE-NEXT: popq %r145780; SSE-NEXT: popq %r155781; SSE-NEXT: popq %rbp5782; SSE-NEXT: retq5783; SSE-NEXT: LBB31_1: ## %cond.store5784; SSE-NEXT: movl (%rsi), %esi5785; SSE-NEXT: movl %esi, (%rdx)5786; SSE-NEXT: testb $2, %dil5787; SSE-NEXT: je LBB31_45788; SSE-NEXT: LBB31_3: ## %cond.store15789; SSE-NEXT: movl %r12d, 4(%rdx)5790; SSE-NEXT: testb $4, %dil5791; SSE-NEXT: je LBB31_65792; SSE-NEXT: LBB31_5: ## %cond.store35793; SSE-NEXT: movl %r15d, 8(%rdx)5794; SSE-NEXT: testb $8, %dil5795; SSE-NEXT: je LBB31_85796; SSE-NEXT: LBB31_7: ## %cond.store55797; SSE-NEXT: movl %r14d, 12(%rdx)5798; SSE-NEXT: testb $16, %dil5799; SSE-NEXT: je LBB31_105800; SSE-NEXT: LBB31_9: ## %cond.store75801; SSE-NEXT: movl %ebp, 16(%rdx)5802; SSE-NEXT: testb $32, %dil5803; SSE-NEXT: je LBB31_125804; SSE-NEXT: LBB31_11: ## %cond.store95805; SSE-NEXT: movl %ebx, 20(%rdx)5806; SSE-NEXT: testb $64, %dil5807; SSE-NEXT: je LBB31_145808; SSE-NEXT: LBB31_13: ## %cond.store115809; SSE-NEXT: movl %r11d, 24(%rdx)5810; SSE-NEXT: testb %dil, %dil5811; SSE-NEXT: jns LBB31_165812; SSE-NEXT: LBB31_15: ## %cond.store135813; SSE-NEXT: movl %r10d, 28(%rdx)5814; SSE-NEXT: testl $256, %edi ## imm = 0x1005815; SSE-NEXT: je LBB31_185816; SSE-NEXT: LBB31_17: ## %cond.store155817; SSE-NEXT: movl %r9d, 32(%rdx)5818; SSE-NEXT: testl $512, %edi ## imm = 0x2005819; SSE-NEXT: je LBB31_205820; SSE-NEXT: LBB31_19: ## %cond.store175821; SSE-NEXT: movl %r8d, 36(%rdx)5822; SSE-NEXT: testl $1024, %edi ## imm = 0x4005823; SSE-NEXT: je LBB31_225824; SSE-NEXT: LBB31_21: ## %cond.store195825; SSE-NEXT: movl %ecx, 40(%rdx)5826; SSE-NEXT: testl $2048, %edi ## imm = 0x8005827; SSE-NEXT: je LBB31_245828; SSE-NEXT: LBB31_23: ## %cond.store215829; SSE-NEXT: movl %eax, 44(%rdx)5830; SSE-NEXT: testl $4096, %edi ## imm = 0x10005831; SSE-NEXT: je LBB31_265832; SSE-NEXT: LBB31_25: ## %cond.store235833; SSE-NEXT: movl %r13d, 48(%rdx)5834; SSE-NEXT: testl $8192, %edi ## imm = 0x20005835; SSE-NEXT: je LBB31_285836; SSE-NEXT: LBB31_27: ## %cond.store255837; SSE-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax ## 4-byte Reload5838; SSE-NEXT: movl %eax, 52(%rdx)5839; SSE-NEXT: testl $16384, %edi ## imm = 0x40005840; SSE-NEXT: je LBB31_305841; SSE-NEXT: LBB31_29: ## %cond.store275842; SSE-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax ## 4-byte Reload5843; SSE-NEXT: movl %eax, 56(%rdx)5844; SSE-NEXT: testw %di, %di5845; SSE-NEXT: jns LBB31_325846; SSE-NEXT: LBB31_31: ## %cond.store295847; SSE-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax ## 4-byte Reload5848; SSE-NEXT: movl %eax, 60(%rdx)5849; SSE-NEXT: testl $65536, %edi ## imm = 0x100005850; SSE-NEXT: je LBB31_345851; SSE-NEXT: LBB31_33: ## %cond.store315852; SSE-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax ## 4-byte Reload5853; SSE-NEXT: movl %eax, 64(%rdx)5854; SSE-NEXT: testl $131072, %edi ## imm = 0x200005855; SSE-NEXT: je LBB31_365856; SSE-NEXT: LBB31_35: ## %cond.store335857; SSE-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax ## 4-byte Reload5858; SSE-NEXT: movl %eax, 68(%rdx)5859; SSE-NEXT: testl $262144, %edi ## imm = 0x400005860; SSE-NEXT: je LBB31_385861; SSE-NEXT: LBB31_37: ## %cond.store355862; SSE-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax ## 4-byte Reload5863; SSE-NEXT: movl %eax, 72(%rdx)5864; SSE-NEXT: testl $524288, %edi ## imm = 0x800005865; SSE-NEXT: je LBB31_405866; SSE-NEXT: LBB31_39: ## %cond.store375867; SSE-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax ## 4-byte Reload5868; SSE-NEXT: movl %eax, 76(%rdx)5869; SSE-NEXT: testl $1048576, %edi ## imm = 0x1000005870; SSE-NEXT: je LBB31_425871; SSE-NEXT: LBB31_41: ## %cond.store395872; SSE-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax ## 4-byte Reload5873; SSE-NEXT: movl %eax, 80(%rdx)5874; SSE-NEXT: testl $2097152, %edi ## imm = 0x2000005875; SSE-NEXT: je LBB31_445876; SSE-NEXT: LBB31_43: ## %cond.store415877; SSE-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax ## 4-byte Reload5878; SSE-NEXT: movl %eax, 84(%rdx)5879; SSE-NEXT: testl $4194304, %edi ## imm = 0x4000005880; SSE-NEXT: jne LBB31_455881; SSE-NEXT: jmp LBB31_465882;5883; AVX1-LABEL: store_v24i32_v24i32_stride6_vf4_only_even_numbered_elts:5884; AVX1: ## %bb.0:5885; AVX1-NEXT: vmovaps (%rsi), %ymm15886; AVX1-NEXT: vmovaps 32(%rsi), %ymm25887; AVX1-NEXT: vmovdqa 64(%rsi), %ymm05888; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm35889; AVX1-NEXT: vpcmpgtd 48(%rdi), %xmm3, %xmm45890; AVX1-NEXT: vpcmpgtd 32(%rdi), %xmm3, %xmm55891; AVX1-NEXT: vpackssdw %xmm4, %xmm5, %xmm45892; AVX1-NEXT: vpacksswb %xmm4, %xmm4, %xmm45893; AVX1-NEXT: vpcmpgtd 80(%rdi), %xmm3, %xmm55894; AVX1-NEXT: vpcmpgtd 64(%rdi), %xmm3, %xmm65895; AVX1-NEXT: vpcmpgtd 16(%rdi), %xmm3, %xmm75896; AVX1-NEXT: vpcmpgtd (%rdi), %xmm3, %xmm85897; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]5898; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero5899; AVX1-NEXT: vpslld $31, %xmm4, %xmm45900; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]5901; AVX1-NEXT: vpslld $31, %xmm3, %xmm35902; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm4, %ymm35903; AVX1-NEXT: vmaskmovps %ymm2, %ymm3, 32(%rdx)5904; AVX1-NEXT: vinsertf128 $1, %xmm7, %ymm8, %ymm25905; AVX1-NEXT: vxorps %xmm3, %xmm3, %xmm35906; AVX1-NEXT: vblendps {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2],ymm3[3],ymm2[4],ymm3[5],ymm2[6],ymm3[7]5907; AVX1-NEXT: vpslld $31, %xmm2, %xmm45908; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm25909; AVX1-NEXT: vpslld $31, %xmm2, %xmm25910; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm4, %ymm25911; AVX1-NEXT: vmaskmovps %ymm1, %ymm2, (%rdx)5912; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm6, %ymm15913; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2],ymm3[3],ymm1[4],ymm3[5],ymm1[6],ymm3[7]5914; AVX1-NEXT: vpslld $31, %xmm1, %xmm25915; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm15916; AVX1-NEXT: vpslld $31, %xmm1, %xmm15917; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm15918; AVX1-NEXT: vmaskmovps %ymm0, %ymm1, 64(%rdx)5919; AVX1-NEXT: vzeroupper5920; AVX1-NEXT: retq5921;5922; AVX2-LABEL: store_v24i32_v24i32_stride6_vf4_only_even_numbered_elts:5923; AVX2: ## %bb.0:5924; AVX2-NEXT: vmovdqa (%rsi), %ymm05925; AVX2-NEXT: vmovdqa 32(%rsi), %ymm15926; AVX2-NEXT: vmovdqa 64(%rsi), %ymm25927; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm35928; AVX2-NEXT: vpcmpgtd 32(%rdi), %ymm3, %ymm45929; AVX2-NEXT: vpcmpgtd (%rdi), %ymm3, %ymm55930; AVX2-NEXT: vpackssdw %ymm4, %ymm5, %ymm45931; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]5932; AVX2-NEXT: vpshufd {{.*#+}} ymm5 = mem[0,2,2,3,4,6,6,7]5933; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,2,2,3]5934; AVX2-NEXT: vpcmpgtd %ymm5, %ymm3, %ymm35935; AVX2-NEXT: vpacksswb %ymm3, %ymm4, %ymm35936; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm3[0,2,1,3]5937; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm45938; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero5939; AVX2-NEXT: vpslld $31, %ymm3, %ymm35940; AVX2-NEXT: vpmaskmovd %ymm0, %ymm3, (%rdx)5941; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm05942; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero5943; AVX2-NEXT: vpslld $31, %ymm0, %ymm05944; AVX2-NEXT: vpmaskmovd %ymm2, %ymm0, 64(%rdx)5945; AVX2-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]5946; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero5947; AVX2-NEXT: vpslld $31, %ymm0, %ymm05948; AVX2-NEXT: vpmaskmovd %ymm1, %ymm0, 32(%rdx)5949; AVX2-NEXT: vzeroupper5950; AVX2-NEXT: retq5951;5952; AVX512F-LABEL: store_v24i32_v24i32_stride6_vf4_only_even_numbered_elts:5953; AVX512F: ## %bb.0:5954; AVX512F-NEXT: vmovdqa64 (%rsi), %zmm05955; AVX512F-NEXT: vmovdqa64 64(%rsi), %zmm15956; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm25957; AVX512F-NEXT: movw $21845, %ax ## imm = 0x55555958; AVX512F-NEXT: kmovw %eax, %k15959; AVX512F-NEXT: vpcmpgtd (%rdi), %zmm2, %k1 {%k1}5960; AVX512F-NEXT: movw $85, %ax5961; AVX512F-NEXT: kmovw %eax, %k25962; AVX512F-NEXT: vpcmpgtd 64(%rdi), %zmm2, %k2 {%k2}5963; AVX512F-NEXT: vmovdqu32 %zmm1, 64(%rdx) {%k2}5964; AVX512F-NEXT: vmovdqu32 %zmm0, (%rdx) {%k1}5965; AVX512F-NEXT: vzeroupper5966; AVX512F-NEXT: retq5967;5968; AVX512VLDQ-LABEL: store_v24i32_v24i32_stride6_vf4_only_even_numbered_elts:5969; AVX512VLDQ: ## %bb.0:5970; AVX512VLDQ-NEXT: vmovdqa64 (%rsi), %zmm05971; AVX512VLDQ-NEXT: vmovdqa64 64(%rsi), %zmm15972; AVX512VLDQ-NEXT: vpxor %xmm2, %xmm2, %xmm25973; AVX512VLDQ-NEXT: movw $21845, %ax ## imm = 0x55555974; AVX512VLDQ-NEXT: kmovw %eax, %k15975; AVX512VLDQ-NEXT: vpcmpgtd (%rdi), %zmm2, %k1 {%k1}5976; AVX512VLDQ-NEXT: movw $85, %ax5977; AVX512VLDQ-NEXT: kmovw %eax, %k25978; AVX512VLDQ-NEXT: vpcmpgtd 64(%rdi), %zmm2, %k2 {%k2}5979; AVX512VLDQ-NEXT: vmovdqu32 %zmm1, 64(%rdx) {%k2}5980; AVX512VLDQ-NEXT: vmovdqu32 %zmm0, (%rdx) {%k1}5981; AVX512VLDQ-NEXT: vzeroupper5982; AVX512VLDQ-NEXT: retq5983;5984; AVX512VLBW-LABEL: store_v24i32_v24i32_stride6_vf4_only_even_numbered_elts:5985; AVX512VLBW: ## %bb.0:5986; AVX512VLBW-NEXT: vmovdqa64 (%rsi), %zmm05987; AVX512VLBW-NEXT: vmovdqa64 64(%rsi), %zmm15988; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm25989; AVX512VLBW-NEXT: movw $21845, %ax ## imm = 0x55555990; AVX512VLBW-NEXT: kmovd %eax, %k15991; AVX512VLBW-NEXT: vpcmpgtd (%rdi), %zmm2, %k1 {%k1}5992; AVX512VLBW-NEXT: movw $85, %ax5993; AVX512VLBW-NEXT: kmovd %eax, %k25994; AVX512VLBW-NEXT: vpcmpgtd 64(%rdi), %zmm2, %k2 {%k2}5995; AVX512VLBW-NEXT: vmovdqu32 %zmm1, 64(%rdx) {%k2}5996; AVX512VLBW-NEXT: vmovdqu32 %zmm0, (%rdx) {%k1}5997; AVX512VLBW-NEXT: vzeroupper5998; AVX512VLBW-NEXT: retq5999;6000; X86-AVX512-LABEL: store_v24i32_v24i32_stride6_vf4_only_even_numbered_elts:6001; X86-AVX512: ## %bb.0:6002; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax6003; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %ecx6004; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %edx6005; X86-AVX512-NEXT: vmovdqa64 (%edx), %zmm06006; X86-AVX512-NEXT: vmovdqa64 64(%edx), %zmm16007; X86-AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm26008; X86-AVX512-NEXT: movw $21845, %dx ## imm = 0x55556009; X86-AVX512-NEXT: kmovd %edx, %k16010; X86-AVX512-NEXT: vpcmpgtd (%ecx), %zmm2, %k1 {%k1}6011; X86-AVX512-NEXT: movw $85, %dx6012; X86-AVX512-NEXT: kmovd %edx, %k26013; X86-AVX512-NEXT: vpcmpgtd 64(%ecx), %zmm2, %k2 {%k2}6014; X86-AVX512-NEXT: vmovdqu32 %zmm1, 64(%eax) {%k2}6015; X86-AVX512-NEXT: vmovdqu32 %zmm0, (%eax) {%k1}6016; X86-AVX512-NEXT: vzeroupper6017; X86-AVX512-NEXT: retl6018 %trigger = load <24 x i32>, ptr %trigger.ptr6019 %val = load <24 x i32>, ptr %val.ptr6020 %mask.src = icmp slt <24 x i32> %trigger, zeroinitializer6021 %mask = and <24 x i1> %mask.src, <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>6022 call void @llvm.masked.store.v24i32.p0(<24 x i32> %val, ptr %dst, i32 1, <24 x i1> %mask)6023 ret void6024}6025 6026; From https://reviews.llvm.org/rGf8d9097168b7#11653116027define void @undefshuffle(<8 x i1> %i0, ptr %src, ptr %dst) nounwind {6028; SSE2-LABEL: undefshuffle:6029; SSE2: ## %bb.0: ## %else6030; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)6031; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax6032; SSE2-NEXT: movd %eax, %xmm06033; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax6034; SSE2-NEXT: pinsrw $1, %eax, %xmm06035; SSE2-NEXT: pinsrw $2, -{{[0-9]+}}(%rsp), %xmm06036; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax6037; SSE2-NEXT: pinsrw $3, %eax, %xmm06038; SSE2-NEXT: psllw $15, %xmm06039; SSE2-NEXT: packsswb %xmm0, %xmm06040; SSE2-NEXT: pmovmskb %xmm0, %eax6041; SSE2-NEXT: testb $1, %al6042; SSE2-NEXT: jne LBB32_16043; SSE2-NEXT: ## %bb.2: ## %else236044; SSE2-NEXT: testb $2, %al6045; SSE2-NEXT: jne LBB32_36046; SSE2-NEXT: LBB32_4: ## %else256047; SSE2-NEXT: testb $4, %al6048; SSE2-NEXT: jne LBB32_56049; SSE2-NEXT: LBB32_6: ## %else276050; SSE2-NEXT: testb $8, %al6051; SSE2-NEXT: jne LBB32_76052; SSE2-NEXT: LBB32_8: ## %else296053; SSE2-NEXT: testb $16, %al6054; SSE2-NEXT: jne LBB32_96055; SSE2-NEXT: LBB32_10: ## %else316056; SSE2-NEXT: testb $32, %al6057; SSE2-NEXT: jne LBB32_116058; SSE2-NEXT: LBB32_12: ## %else336059; SSE2-NEXT: testb $64, %al6060; SSE2-NEXT: jne LBB32_136061; SSE2-NEXT: LBB32_14: ## %else356062; SSE2-NEXT: testb $-128, %al6063; SSE2-NEXT: jne LBB32_156064; SSE2-NEXT: LBB32_16: ## %else376065; SSE2-NEXT: retq6066; SSE2-NEXT: LBB32_1: ## %cond.store6067; SSE2-NEXT: movl $0, (%rsi)6068; SSE2-NEXT: testb $2, %al6069; SSE2-NEXT: je LBB32_46070; SSE2-NEXT: LBB32_3: ## %cond.store246071; SSE2-NEXT: movl $0, 4(%rsi)6072; SSE2-NEXT: testb $4, %al6073; SSE2-NEXT: je LBB32_66074; SSE2-NEXT: LBB32_5: ## %cond.store266075; SSE2-NEXT: movl $0, 8(%rsi)6076; SSE2-NEXT: testb $8, %al6077; SSE2-NEXT: je LBB32_86078; SSE2-NEXT: LBB32_7: ## %cond.store286079; SSE2-NEXT: movl $0, 12(%rsi)6080; SSE2-NEXT: testb $16, %al6081; SSE2-NEXT: je LBB32_106082; SSE2-NEXT: LBB32_9: ## %cond.store306083; SSE2-NEXT: movl $0, 16(%rsi)6084; SSE2-NEXT: testb $32, %al6085; SSE2-NEXT: je LBB32_126086; SSE2-NEXT: LBB32_11: ## %cond.store326087; SSE2-NEXT: movl $0, 20(%rsi)6088; SSE2-NEXT: testb $64, %al6089; SSE2-NEXT: je LBB32_146090; SSE2-NEXT: LBB32_13: ## %cond.store346091; SSE2-NEXT: movl $0, 24(%rsi)6092; SSE2-NEXT: testb $-128, %al6093; SSE2-NEXT: je LBB32_166094; SSE2-NEXT: LBB32_15: ## %cond.store366095; SSE2-NEXT: movl $0, 28(%rsi)6096; SSE2-NEXT: retq6097;6098; SSE4-LABEL: undefshuffle:6099; SSE4: ## %bb.0: ## %else6100; SSE4-NEXT: psllw $15, %xmm06101; SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm06102; SSE4-NEXT: packsswb %xmm0, %xmm06103; SSE4-NEXT: pmovmskb %xmm0, %eax6104; SSE4-NEXT: testb $1, %al6105; SSE4-NEXT: jne LBB32_16106; SSE4-NEXT: ## %bb.2: ## %else236107; SSE4-NEXT: testb $2, %al6108; SSE4-NEXT: jne LBB32_36109; SSE4-NEXT: LBB32_4: ## %else256110; SSE4-NEXT: testb $4, %al6111; SSE4-NEXT: jne LBB32_56112; SSE4-NEXT: LBB32_6: ## %else276113; SSE4-NEXT: testb $8, %al6114; SSE4-NEXT: jne LBB32_76115; SSE4-NEXT: LBB32_8: ## %else296116; SSE4-NEXT: testb $16, %al6117; SSE4-NEXT: jne LBB32_96118; SSE4-NEXT: LBB32_10: ## %else316119; SSE4-NEXT: testb $32, %al6120; SSE4-NEXT: jne LBB32_116121; SSE4-NEXT: LBB32_12: ## %else336122; SSE4-NEXT: testb $64, %al6123; SSE4-NEXT: jne LBB32_136124; SSE4-NEXT: LBB32_14: ## %else356125; SSE4-NEXT: testb $-128, %al6126; SSE4-NEXT: jne LBB32_156127; SSE4-NEXT: LBB32_16: ## %else376128; SSE4-NEXT: retq6129; SSE4-NEXT: LBB32_1: ## %cond.store6130; SSE4-NEXT: movl $0, (%rsi)6131; SSE4-NEXT: testb $2, %al6132; SSE4-NEXT: je LBB32_46133; SSE4-NEXT: LBB32_3: ## %cond.store246134; SSE4-NEXT: movl $0, 4(%rsi)6135; SSE4-NEXT: testb $4, %al6136; SSE4-NEXT: je LBB32_66137; SSE4-NEXT: LBB32_5: ## %cond.store266138; SSE4-NEXT: movl $0, 8(%rsi)6139; SSE4-NEXT: testb $8, %al6140; SSE4-NEXT: je LBB32_86141; SSE4-NEXT: LBB32_7: ## %cond.store286142; SSE4-NEXT: movl $0, 12(%rsi)6143; SSE4-NEXT: testb $16, %al6144; SSE4-NEXT: je LBB32_106145; SSE4-NEXT: LBB32_9: ## %cond.store306146; SSE4-NEXT: movl $0, 16(%rsi)6147; SSE4-NEXT: testb $32, %al6148; SSE4-NEXT: je LBB32_126149; SSE4-NEXT: LBB32_11: ## %cond.store326150; SSE4-NEXT: movl $0, 20(%rsi)6151; SSE4-NEXT: testb $64, %al6152; SSE4-NEXT: je LBB32_146153; SSE4-NEXT: LBB32_13: ## %cond.store346154; SSE4-NEXT: movl $0, 24(%rsi)6155; SSE4-NEXT: testb $-128, %al6156; SSE4-NEXT: je LBB32_166157; SSE4-NEXT: LBB32_15: ## %cond.store366158; SSE4-NEXT: movl $0, 28(%rsi)6159; SSE4-NEXT: retq6160;6161; AVX1-LABEL: undefshuffle:6162; AVX1: ## %bb.0:6163; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero6164; AVX1-NEXT: vpslld $31, %xmm0, %xmm06165; AVX1-NEXT: vxorps %xmm1, %xmm1, %xmm16166; AVX1-NEXT: vmaskmovps %ymm1, %ymm0, (%rsi)6167; AVX1-NEXT: vzeroupper6168; AVX1-NEXT: retq6169;6170; AVX2-LABEL: undefshuffle:6171; AVX2: ## %bb.0:6172; AVX2-NEXT: ## kill: def $xmm0 killed $xmm0 def $ymm06173; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,u,u,u,2,u,u,u,4,u,u,u,6,u,u,u],zero,ymm0[u,u,u],zero,ymm0[u,u,u],zero,ymm0[u,u,u],zero,ymm0[u,u,u]6174; AVX2-NEXT: vpslld $31, %ymm0, %ymm06175; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm16176; AVX2-NEXT: vpmaskmovd %ymm1, %ymm0, (%rsi)6177; AVX2-NEXT: vzeroupper6178; AVX2-NEXT: retq6179;6180; AVX512F-LABEL: undefshuffle:6181; AVX512F: ## %bb.0:6182; AVX512F-NEXT: vpmovsxwq %xmm0, %zmm06183; AVX512F-NEXT: vpsllq $63, %zmm0, %zmm06184; AVX512F-NEXT: movb $15, %al6185; AVX512F-NEXT: kmovw %eax, %k16186; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 {%k1}6187; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm06188; AVX512F-NEXT: vmovdqu32 %zmm0, (%rsi) {%k1}6189; AVX512F-NEXT: vzeroupper6190; AVX512F-NEXT: retq6191;6192; AVX512VLDQ-LABEL: undefshuffle:6193; AVX512VLDQ: ## %bb.0:6194; AVX512VLDQ-NEXT: vpmovsxwd %xmm0, %ymm06195; AVX512VLDQ-NEXT: vpslld $31, %ymm0, %ymm06196; AVX512VLDQ-NEXT: vpxor %xmm1, %xmm1, %xmm16197; AVX512VLDQ-NEXT: movb $15, %al6198; AVX512VLDQ-NEXT: kmovw %eax, %k16199; AVX512VLDQ-NEXT: vpcmpgtd %ymm0, %ymm1, %k1 {%k1}6200; AVX512VLDQ-NEXT: vmovdqu32 %ymm1, (%rsi) {%k1}6201; AVX512VLDQ-NEXT: vzeroupper6202; AVX512VLDQ-NEXT: retq6203;6204; AVX512VLBW-LABEL: undefshuffle:6205; AVX512VLBW: ## %bb.0:6206; AVX512VLBW-NEXT: vpsllw $15, %xmm0, %xmm06207; AVX512VLBW-NEXT: vpmovw2m %xmm0, %k06208; AVX512VLBW-NEXT: movl $15, %eax6209; AVX512VLBW-NEXT: kmovd %eax, %k16210; AVX512VLBW-NEXT: kandd %k1, %k0, %k16211; AVX512VLBW-NEXT: vpxor %xmm0, %xmm0, %xmm06212; AVX512VLBW-NEXT: vmovdqu32 %ymm0, (%rsi) {%k1}6213; AVX512VLBW-NEXT: vzeroupper6214; AVX512VLBW-NEXT: retq6215;6216; X86-AVX512-LABEL: undefshuffle:6217; X86-AVX512: ## %bb.0:6218; X86-AVX512-NEXT: vpsllw $15, %xmm0, %xmm06219; X86-AVX512-NEXT: vpmovw2m %xmm0, %k06220; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax6221; X86-AVX512-NEXT: movl $15, %ecx6222; X86-AVX512-NEXT: kmovd %ecx, %k16223; X86-AVX512-NEXT: kandd %k1, %k0, %k16224; X86-AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm06225; X86-AVX512-NEXT: vmovdqu32 %ymm0, (%eax) {%k1}6226; X86-AVX512-NEXT: vzeroupper6227; X86-AVX512-NEXT: retl6228 %i1 = shufflevector <8 x i1> %i0, <8 x i1> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>6229 %i2 = shufflevector <16 x i1> %i1, <16 x i1> zeroinitializer, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>6230 %it51 = and <32 x i1> %i2, <i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>6231 %i3 = shufflevector <32 x i1> %it51, <32 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>6232 %i4 = shufflevector <32 x i1> %it51, <32 x i1> zeroinitializer, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>6233 %i5 = tail call <8 x i32> @llvm.masked.load.v8i32.p0(ptr %src, i32 1, <8 x i1> %i4, <8 x i32> zeroinitializer)6234 tail call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr %dst, i32 1, <8 x i1> %i3)6235 ret void6236}6237declare <8 x i32> @llvm.masked.load.v8i32.p0(ptr nocapture, i32 immarg, <8 x i1>, <8 x i32>)6238 6239declare void @llvm.masked.store.v8f64.p0(<8 x double>, ptr, i32, <8 x i1>)6240declare void @llvm.masked.store.v4f64.p0(<4 x double>, ptr, i32, <4 x i1>)6241declare void @llvm.masked.store.v2f64.p0(<2 x double>, ptr, i32, <2 x i1>)6242declare void @llvm.masked.store.v1f64.p0(<1 x double>, ptr, i32, <1 x i1>)6243 6244declare void @llvm.masked.store.v16f32.p0(<16 x float>, ptr, i32, <16 x i1>)6245declare void @llvm.masked.store.v8f32.p0(<8 x float>, ptr, i32, <8 x i1>)6246declare void @llvm.masked.store.v4f32.p0(<4 x float>, ptr, i32, <4 x i1>)6247declare void @llvm.masked.store.v2f32.p0(<2 x float>, ptr, i32, <2 x i1>)6248 6249declare void @llvm.masked.store.v16i64.p0(<16 x i64>, ptr, i32, <16 x i1>)6250declare void @llvm.masked.store.v8i64.p0(<8 x i64>, ptr, i32, <8 x i1>)6251declare void @llvm.masked.store.v4i64.p0(<4 x i64>, ptr, i32, <4 x i1>)6252declare void @llvm.masked.store.v2i64.p0(<2 x i64>, ptr, i32, <2 x i1>)6253declare void @llvm.masked.store.v1i64.p0(<1 x i64>, ptr, i32, <1 x i1>)6254 6255declare void @llvm.masked.store.v24i32.p0(<24 x i32>, ptr, i32, <24 x i1>)6256declare void @llvm.masked.store.v16i32.p0(<16 x i32>, ptr, i32, <16 x i1>)6257declare void @llvm.masked.store.v8i32.p0(<8 x i32>, ptr, i32, <8 x i1>)6258declare void @llvm.masked.store.v4i32.p0(<4 x i32>, ptr, i32, <4 x i1>)6259declare void @llvm.masked.store.v3i32.p0(<3 x i32>, ptr, i32, <3 x i1>)6260declare void @llvm.masked.store.v2i32.p0(<2 x i32>, ptr, i32, <2 x i1>)6261declare void @llvm.masked.store.v1i32.p0(<1 x i32>, ptr, i32, <1 x i1>)6262 6263declare void @llvm.masked.store.v32i16.p0(<32 x i16>, ptr, i32, <32 x i1>)6264declare void @llvm.masked.store.v16i16.p0(<16 x i16>, ptr, i32, <16 x i1>)6265declare void @llvm.masked.store.v8i16.p0(<8 x i16>, ptr, i32, <8 x i1>)6266declare void @llvm.masked.store.v4i16.p0(<4 x i16>, ptr, i32, <4 x i1>)6267 6268declare void @llvm.masked.store.v64i8.p0(<64 x i8>, ptr, i32, <64 x i1>)6269declare void @llvm.masked.store.v32i8.p0(<32 x i8>, ptr, i32, <32 x i1>)6270declare void @llvm.masked.store.v16i8.p0(<16 x i8>, ptr, i32, <16 x i1>)6271declare void @llvm.masked.store.v8i8.p0(<8 x i8>, ptr, i32, <8 x i1>)6272