3807 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE3; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX4; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX25; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FP6; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FCP7; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX5128; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512-FCP9; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ10; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-FCP11; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW12; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW-FCP13; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512DQ-BW14; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-BW-FCP15 16; These patterns are produced by LoopVectorizer for interleaved stores.17 18define void @store_i16_stride3_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %out.vec) nounwind {19; SSE-LABEL: store_i16_stride3_vf2:20; SSE: # %bb.0:21; SSE-NEXT: movdqa (%rdi), %xmm022; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]23; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]24; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,1]25; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,6,7]26; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]27; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,1,3,4,5,6,7]28; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,7,6,7]29; SSE-NEXT: movq %xmm0, (%rcx)30; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]31; SSE-NEXT: movd %xmm0, 8(%rcx)32; SSE-NEXT: retq33;34; AVX-LABEL: store_i16_stride3_vf2:35; AVX: # %bb.0:36; AVX-NEXT: vmovdqa (%rdi), %xmm037; AVX-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]38; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]39; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,4,5,12,13,6,7,4,5,6,7]40; AVX-NEXT: vpextrd $2, %xmm0, 8(%rcx)41; AVX-NEXT: vmovq %xmm0, (%rcx)42; AVX-NEXT: retq43;44; AVX2-LABEL: store_i16_stride3_vf2:45; AVX2: # %bb.0:46; AVX2-NEXT: vmovdqa (%rdi), %xmm047; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]48; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]49; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,4,5,12,13,6,7,4,5,6,7]50; AVX2-NEXT: vpextrd $2, %xmm0, 8(%rcx)51; AVX2-NEXT: vmovq %xmm0, (%rcx)52; AVX2-NEXT: retq53;54; AVX2-FP-LABEL: store_i16_stride3_vf2:55; AVX2-FP: # %bb.0:56; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm057; AVX2-FP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]58; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]59; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,4,5,12,13,6,7,4,5,6,7]60; AVX2-FP-NEXT: vpextrd $2, %xmm0, 8(%rcx)61; AVX2-FP-NEXT: vmovq %xmm0, (%rcx)62; AVX2-FP-NEXT: retq63;64; AVX2-FCP-LABEL: store_i16_stride3_vf2:65; AVX2-FCP: # %bb.0:66; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm067; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]68; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]69; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,4,5,12,13,6,7,4,5,6,7]70; AVX2-FCP-NEXT: vpextrd $2, %xmm0, 8(%rcx)71; AVX2-FCP-NEXT: vmovq %xmm0, (%rcx)72; AVX2-FCP-NEXT: retq73;74; AVX512-LABEL: store_i16_stride3_vf2:75; AVX512: # %bb.0:76; AVX512-NEXT: vmovdqa (%rdi), %xmm077; AVX512-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]78; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]79; AVX512-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,4,5,12,13,6,7,4,5,6,7]80; AVX512-NEXT: vpextrd $2, %xmm0, 8(%rcx)81; AVX512-NEXT: vmovq %xmm0, (%rcx)82; AVX512-NEXT: retq83;84; AVX512-FCP-LABEL: store_i16_stride3_vf2:85; AVX512-FCP: # %bb.0:86; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm087; AVX512-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]88; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]89; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,4,5,12,13,6,7,4,5,6,7]90; AVX512-FCP-NEXT: vpextrd $2, %xmm0, 8(%rcx)91; AVX512-FCP-NEXT: vmovq %xmm0, (%rcx)92; AVX512-FCP-NEXT: retq93;94; AVX512DQ-LABEL: store_i16_stride3_vf2:95; AVX512DQ: # %bb.0:96; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm097; AVX512DQ-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]98; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]99; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,4,5,12,13,6,7,4,5,6,7]100; AVX512DQ-NEXT: vpextrd $2, %xmm0, 8(%rcx)101; AVX512DQ-NEXT: vmovq %xmm0, (%rcx)102; AVX512DQ-NEXT: retq103;104; AVX512DQ-FCP-LABEL: store_i16_stride3_vf2:105; AVX512DQ-FCP: # %bb.0:106; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0107; AVX512DQ-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]108; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]109; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,4,5,12,13,6,7,4,5,6,7]110; AVX512DQ-FCP-NEXT: vpextrd $2, %xmm0, 8(%rcx)111; AVX512DQ-FCP-NEXT: vmovq %xmm0, (%rcx)112; AVX512DQ-FCP-NEXT: retq113;114; AVX512BW-LABEL: store_i16_stride3_vf2:115; AVX512BW: # %bb.0:116; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0117; AVX512BW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]118; AVX512BW-NEXT: vpmovsxbw {{.*#+}} xmm1 = [0,2,8,1,3,9,1,9]119; AVX512BW-NEXT: vpermi2w (%rdx), %xmm0, %xmm1120; AVX512BW-NEXT: vpextrd $2, %xmm1, 8(%rcx)121; AVX512BW-NEXT: vmovq %xmm1, (%rcx)122; AVX512BW-NEXT: retq123;124; AVX512BW-FCP-LABEL: store_i16_stride3_vf2:125; AVX512BW-FCP: # %bb.0:126; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm0127; AVX512BW-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]128; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm1 = [0,2,8,1,3,9,1,9]129; AVX512BW-FCP-NEXT: vpermi2w (%rdx), %xmm0, %xmm1130; AVX512BW-FCP-NEXT: vpextrd $2, %xmm1, 8(%rcx)131; AVX512BW-FCP-NEXT: vmovq %xmm1, (%rcx)132; AVX512BW-FCP-NEXT: retq133;134; AVX512DQ-BW-LABEL: store_i16_stride3_vf2:135; AVX512DQ-BW: # %bb.0:136; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm0137; AVX512DQ-BW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]138; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} xmm1 = [0,2,8,1,3,9,1,9]139; AVX512DQ-BW-NEXT: vpermi2w (%rdx), %xmm0, %xmm1140; AVX512DQ-BW-NEXT: vpextrd $2, %xmm1, 8(%rcx)141; AVX512DQ-BW-NEXT: vmovq %xmm1, (%rcx)142; AVX512DQ-BW-NEXT: retq143;144; AVX512DQ-BW-FCP-LABEL: store_i16_stride3_vf2:145; AVX512DQ-BW-FCP: # %bb.0:146; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm0147; AVX512DQ-BW-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]148; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} xmm1 = [0,2,8,1,3,9,1,9]149; AVX512DQ-BW-FCP-NEXT: vpermi2w (%rdx), %xmm0, %xmm1150; AVX512DQ-BW-FCP-NEXT: vpextrd $2, %xmm1, 8(%rcx)151; AVX512DQ-BW-FCP-NEXT: vmovq %xmm1, (%rcx)152; AVX512DQ-BW-FCP-NEXT: retq153 %in.vec0 = load <2 x i16>, ptr %in.vecptr0, align 64154 %in.vec1 = load <2 x i16>, ptr %in.vecptr1, align 64155 %in.vec2 = load <2 x i16>, ptr %in.vecptr2, align 64156 %1 = shufflevector <2 x i16> %in.vec0, <2 x i16> %in.vec1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>157 %2 = shufflevector <2 x i16> %in.vec2, <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>158 %3 = shufflevector <4 x i16> %1, <4 x i16> %2, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>159 %interleaved.vec = shufflevector <6 x i16> %3, <6 x i16> poison, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>160 store <6 x i16> %interleaved.vec, ptr %out.vec, align 64161 ret void162}163 164define void @store_i16_stride3_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %out.vec) nounwind {165; SSE-LABEL: store_i16_stride3_vf4:166; SSE: # %bb.0:167; SSE-NEXT: movq {{.*#+}} xmm0 = mem[0],zero168; SSE-NEXT: movq {{.*#+}} xmm1 = mem[0],zero169; SSE-NEXT: movq {{.*#+}} xmm2 = mem[0],zero170; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]171; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]172; SSE-NEXT: movdqa {{.*#+}} xmm3 = [65535,65535,0,65535,65535,0,65535,65535]173; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm2[0,1,2,3,6,5,4,7]174; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,3,2,1]175; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,2,2,1,4,5,6,7]176; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,6,4]177; SSE-NEXT: pand %xmm3, %xmm4178; SSE-NEXT: pandn %xmm1, %xmm3179; SSE-NEXT: por %xmm4, %xmm3180; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]181; SSE-NEXT: movdqa {{.*#+}} xmm1 = [0,65535,65535,0,65535,65535,65535,65535]182; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]183; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,3,1,3,4,5,6,7]184; SSE-NEXT: pand %xmm1, %xmm2185; SSE-NEXT: pandn %xmm0, %xmm1186; SSE-NEXT: por %xmm2, %xmm1187; SSE-NEXT: movq %xmm1, 16(%rcx)188; SSE-NEXT: movdqa %xmm3, (%rcx)189; SSE-NEXT: retq190;191; AVX-LABEL: store_i16_stride3_vf4:192; AVX: # %bb.0:193; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero194; AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero195; AVX-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero196; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]197; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]198; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,3,1,3,4,5,6,7]199; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]200; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0],xmm2[1,2],xmm3[3],xmm2[4,5,6,7]201; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,1,8,9,u,u,2,3,10,11,u,u,4,5,12,13]202; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]203; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]204; AVX-NEXT: vmovdqa %xmm0, (%rcx)205; AVX-NEXT: vmovq %xmm2, 16(%rcx)206; AVX-NEXT: retq207;208; AVX2-LABEL: store_i16_stride3_vf4:209; AVX2: # %bb.0:210; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero211; AVX2-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero212; AVX2-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero213; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]214; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm2215; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0216; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,ymm0[0,1],zero,zero,zero,zero,ymm0[2,3],zero,zero,zero,zero,zero,zero,ymm0[22,23,30,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero217; AVX2-NEXT: vpshufb {{.*#+}} ymm1 = ymm2[0,1,8,9],zero,zero,ymm2[2,3,10,11],zero,zero,ymm2[4,5,12,13,20,21],zero,zero,zero,zero,ymm2[22,23],zero,zero,zero,zero,zero,zero,zero,zero218; AVX2-NEXT: vpor %ymm0, %ymm1, %ymm0219; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1220; AVX2-NEXT: vmovq %xmm1, 16(%rcx)221; AVX2-NEXT: vmovdqa %xmm0, (%rcx)222; AVX2-NEXT: vzeroupper223; AVX2-NEXT: retq224;225; AVX2-FP-LABEL: store_i16_stride3_vf4:226; AVX2-FP: # %bb.0:227; AVX2-FP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero228; AVX2-FP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero229; AVX2-FP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero230; AVX2-FP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]231; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm2232; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0233; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,ymm0[0,1],zero,zero,zero,zero,ymm0[2,3],zero,zero,zero,zero,zero,zero,ymm0[22,23,30,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero234; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm1 = ymm2[0,1,8,9],zero,zero,ymm2[2,3,10,11],zero,zero,ymm2[4,5,12,13,20,21],zero,zero,zero,zero,ymm2[22,23],zero,zero,zero,zero,zero,zero,zero,zero235; AVX2-FP-NEXT: vpor %ymm0, %ymm1, %ymm0236; AVX2-FP-NEXT: vextracti128 $1, %ymm0, %xmm1237; AVX2-FP-NEXT: vmovq %xmm1, 16(%rcx)238; AVX2-FP-NEXT: vmovdqa %xmm0, (%rcx)239; AVX2-FP-NEXT: vzeroupper240; AVX2-FP-NEXT: retq241;242; AVX2-FCP-LABEL: store_i16_stride3_vf4:243; AVX2-FCP: # %bb.0:244; AVX2-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero245; AVX2-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero246; AVX2-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero247; AVX2-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]248; AVX2-FCP-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm2249; AVX2-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0250; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,ymm0[0,1],zero,zero,zero,zero,ymm0[2,3],zero,zero,zero,zero,zero,zero,ymm0[22,23,30,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero251; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm2[0,1,8,9],zero,zero,ymm2[2,3,10,11],zero,zero,ymm2[4,5,12,13,20,21],zero,zero,zero,zero,ymm2[22,23],zero,zero,zero,zero,zero,zero,zero,zero252; AVX2-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0253; AVX2-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1254; AVX2-FCP-NEXT: vmovq %xmm1, 16(%rcx)255; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rcx)256; AVX2-FCP-NEXT: vzeroupper257; AVX2-FCP-NEXT: retq258;259; AVX512-LABEL: store_i16_stride3_vf4:260; AVX512: # %bb.0:261; AVX512-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero262; AVX512-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero263; AVX512-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero264; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]265; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm2266; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0267; AVX512-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,ymm0[0,1],zero,zero,zero,zero,ymm0[2,3],zero,zero,zero,zero,zero,zero,ymm0[22,23,30,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero268; AVX512-NEXT: vpshufb {{.*#+}} ymm1 = ymm2[0,1,8,9],zero,zero,ymm2[2,3,10,11],zero,zero,ymm2[4,5,12,13,20,21],zero,zero,zero,zero,ymm2[22,23,u,u,u,u,u,u,u,u]269; AVX512-NEXT: vpor %ymm0, %ymm1, %ymm0270; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1271; AVX512-NEXT: vmovq %xmm1, 16(%rcx)272; AVX512-NEXT: vmovdqa %xmm0, (%rcx)273; AVX512-NEXT: vzeroupper274; AVX512-NEXT: retq275;276; AVX512-FCP-LABEL: store_i16_stride3_vf4:277; AVX512-FCP: # %bb.0:278; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero279; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero280; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero281; AVX512-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]282; AVX512-FCP-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm2283; AVX512-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0284; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,ymm0[0,1],zero,zero,zero,zero,ymm0[2,3],zero,zero,zero,zero,zero,zero,ymm0[22,23,30,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero285; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm2[0,1,8,9],zero,zero,ymm2[2,3,10,11],zero,zero,ymm2[4,5,12,13,20,21],zero,zero,zero,zero,ymm2[22,23,u,u,u,u,u,u,u,u]286; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0287; AVX512-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1288; AVX512-FCP-NEXT: vmovq %xmm1, 16(%rcx)289; AVX512-FCP-NEXT: vmovdqa %xmm0, (%rcx)290; AVX512-FCP-NEXT: vzeroupper291; AVX512-FCP-NEXT: retq292;293; AVX512DQ-LABEL: store_i16_stride3_vf4:294; AVX512DQ: # %bb.0:295; AVX512DQ-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero296; AVX512DQ-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero297; AVX512DQ-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero298; AVX512DQ-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]299; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm2300; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0301; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,ymm0[0,1],zero,zero,zero,zero,ymm0[2,3],zero,zero,zero,zero,zero,zero,ymm0[22,23,30,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero302; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm1 = ymm2[0,1,8,9],zero,zero,ymm2[2,3,10,11],zero,zero,ymm2[4,5,12,13,20,21],zero,zero,zero,zero,ymm2[22,23,u,u,u,u,u,u,u,u]303; AVX512DQ-NEXT: vpor %ymm0, %ymm1, %ymm0304; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1305; AVX512DQ-NEXT: vmovq %xmm1, 16(%rcx)306; AVX512DQ-NEXT: vmovdqa %xmm0, (%rcx)307; AVX512DQ-NEXT: vzeroupper308; AVX512DQ-NEXT: retq309;310; AVX512DQ-FCP-LABEL: store_i16_stride3_vf4:311; AVX512DQ-FCP: # %bb.0:312; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero313; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero314; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero315; AVX512DQ-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]316; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm2317; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0318; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,ymm0[0,1],zero,zero,zero,zero,ymm0[2,3],zero,zero,zero,zero,zero,zero,ymm0[22,23,30,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero319; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm2[0,1,8,9],zero,zero,ymm2[2,3,10,11],zero,zero,ymm2[4,5,12,13,20,21],zero,zero,zero,zero,ymm2[22,23,u,u,u,u,u,u,u,u]320; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm0321; AVX512DQ-FCP-NEXT: vextracti128 $1, %ymm0, %xmm1322; AVX512DQ-FCP-NEXT: vmovq %xmm1, 16(%rcx)323; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, (%rcx)324; AVX512DQ-FCP-NEXT: vzeroupper325; AVX512DQ-FCP-NEXT: retq326;327; AVX512BW-LABEL: store_i16_stride3_vf4:328; AVX512BW: # %bb.0:329; AVX512BW-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero330; AVX512BW-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero331; AVX512BW-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero332; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]333; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,4,16,1,5,17,2,6,18,3,7,19,0,0,0,0]334; AVX512BW-NEXT: vpermi2w %ymm0, %ymm1, %ymm2335; AVX512BW-NEXT: vextracti128 $1, %ymm2, %xmm0336; AVX512BW-NEXT: vmovq %xmm0, 16(%rcx)337; AVX512BW-NEXT: vmovdqa %xmm2, (%rcx)338; AVX512BW-NEXT: vzeroupper339; AVX512BW-NEXT: retq340;341; AVX512BW-FCP-LABEL: store_i16_stride3_vf4:342; AVX512BW-FCP: # %bb.0:343; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero344; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero345; AVX512BW-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero346; AVX512BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]347; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,4,16,1,5,17,2,6,18,3,7,19,0,0,0,0]348; AVX512BW-FCP-NEXT: vpermi2w %ymm0, %ymm1, %ymm2349; AVX512BW-FCP-NEXT: vextracti128 $1, %ymm2, %xmm0350; AVX512BW-FCP-NEXT: vmovq %xmm0, 16(%rcx)351; AVX512BW-FCP-NEXT: vmovdqa %xmm2, (%rcx)352; AVX512BW-FCP-NEXT: vzeroupper353; AVX512BW-FCP-NEXT: retq354;355; AVX512DQ-BW-LABEL: store_i16_stride3_vf4:356; AVX512DQ-BW: # %bb.0:357; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero358; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero359; AVX512DQ-BW-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero360; AVX512DQ-BW-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]361; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,4,16,1,5,17,2,6,18,3,7,19,0,0,0,0]362; AVX512DQ-BW-NEXT: vpermi2w %ymm0, %ymm1, %ymm2363; AVX512DQ-BW-NEXT: vextracti128 $1, %ymm2, %xmm0364; AVX512DQ-BW-NEXT: vmovq %xmm0, 16(%rcx)365; AVX512DQ-BW-NEXT: vmovdqa %xmm2, (%rcx)366; AVX512DQ-BW-NEXT: vzeroupper367; AVX512DQ-BW-NEXT: retq368;369; AVX512DQ-BW-FCP-LABEL: store_i16_stride3_vf4:370; AVX512DQ-BW-FCP: # %bb.0:371; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero372; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero373; AVX512DQ-BW-FCP-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero374; AVX512DQ-BW-FCP-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]375; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [0,4,16,1,5,17,2,6,18,3,7,19,0,0,0,0]376; AVX512DQ-BW-FCP-NEXT: vpermi2w %ymm0, %ymm1, %ymm2377; AVX512DQ-BW-FCP-NEXT: vextracti128 $1, %ymm2, %xmm0378; AVX512DQ-BW-FCP-NEXT: vmovq %xmm0, 16(%rcx)379; AVX512DQ-BW-FCP-NEXT: vmovdqa %xmm2, (%rcx)380; AVX512DQ-BW-FCP-NEXT: vzeroupper381; AVX512DQ-BW-FCP-NEXT: retq382 %in.vec0 = load <4 x i16>, ptr %in.vecptr0, align 64383 %in.vec1 = load <4 x i16>, ptr %in.vecptr1, align 64384 %in.vec2 = load <4 x i16>, ptr %in.vecptr2, align 64385 %1 = shufflevector <4 x i16> %in.vec0, <4 x i16> %in.vec1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>386 %2 = shufflevector <4 x i16> %in.vec2, <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>387 %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>388 %interleaved.vec = shufflevector <12 x i16> %3, <12 x i16> poison, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>389 store <12 x i16> %interleaved.vec, ptr %out.vec, align 64390 ret void391}392 393define void @store_i16_stride3_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %out.vec) nounwind {394; SSE-LABEL: store_i16_stride3_vf8:395; SSE: # %bb.0:396; SSE-NEXT: movdqa (%rdi), %xmm0397; SSE-NEXT: movdqa (%rsi), %xmm2398; SSE-NEXT: movdqa (%rdx), %xmm3399; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,0,0,0]400; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,0,65535,65535,0,65535,65535]401; SSE-NEXT: movdqa %xmm4, %xmm5402; SSE-NEXT: pandn %xmm1, %xmm5403; SSE-NEXT: movdqa %xmm0, %xmm1404; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]405; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]406; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,2,4,5,6,7]407; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,4,5]408; SSE-NEXT: pand %xmm4, %xmm1409; SSE-NEXT: por %xmm5, %xmm1410; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,2,2]411; SSE-NEXT: pand %xmm4, %xmm5412; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm2[3,3,3,3,4,5,6,7]413; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,4,4]414; SSE-NEXT: pandn %xmm6, %xmm4415; SSE-NEXT: por %xmm5, %xmm4416; SSE-NEXT: movdqa {{.*#+}} xmm5 = [0,65535,65535,0,65535,65535,0,65535]417; SSE-NEXT: pand %xmm5, %xmm4418; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm3[1,1,2,2]419; SSE-NEXT: pandn %xmm6, %xmm5420; SSE-NEXT: por %xmm4, %xmm5421; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,2,3,3]422; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,0,65535,65535,0,65535,65535,0]423; SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]424; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,1,3,3]425; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,1,0,4,5,6,7]426; SSE-NEXT: pand %xmm4, %xmm0427; SSE-NEXT: pandn %xmm3, %xmm4428; SSE-NEXT: por %xmm0, %xmm4429; SSE-NEXT: movdqa %xmm4, 32(%rcx)430; SSE-NEXT: movdqa %xmm5, 16(%rcx)431; SSE-NEXT: movdqa %xmm1, (%rcx)432; SSE-NEXT: retq433;434; AVX-LABEL: store_i16_stride3_vf8:435; AVX: # %bb.0:436; AVX-NEXT: vmovdqa (%rdi), %xmm0437; AVX-NEXT: vmovdqa (%rsi), %xmm1438; AVX-NEXT: vmovdqa (%rdx), %xmm2439; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,2]440; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm1[3,3,3,3,4,5,6,7]441; AVX-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,4,4,4]442; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2],xmm3[3,4],xmm4[5],xmm3[6,7]443; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,2,2]444; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1,2],xmm4[3],xmm3[4,5],xmm4[6],xmm3[7]445; AVX-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]446; AVX-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[4,5,u,u,10,11,8,9,u,u,14,15,12,13,u,u]447; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,2,3,3]448; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1],xmm4[2,3],xmm5[4],xmm4[5,6],xmm5[7]449; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]450; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,u,u,4,5,6,7,u,u,8,9,10,11]451; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[0,0,0,0]452; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]453; AVX-NEXT: vmovdqa %xmm0, (%rcx)454; AVX-NEXT: vmovdqa %xmm4, 32(%rcx)455; AVX-NEXT: vmovdqa %xmm3, 16(%rcx)456; AVX-NEXT: retq457;458; AVX2-LABEL: store_i16_stride3_vf8:459; AVX2: # %bb.0:460; AVX2-NEXT: vmovdqa (%rdi), %xmm0461; AVX2-NEXT: vmovdqa (%rsi), %xmm1462; AVX2-NEXT: vmovdqa (%rdx), %xmm2463; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm3464; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,0,1,6,7,2,3,2,3,8,9,4,5,4,5,16,17,6,7,22,23,18,19,8,9,24,25,20,21,10,11]465; AVX2-NEXT: vpshufb %ymm4, %ymm3, %ymm5466; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,0,1]467; AVX2-NEXT: vpshufb %ymm4, %ymm3, %ymm3468; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm5[0],ymm3[1],ymm5[2,3],ymm3[4],ymm5[5,6],ymm3[7],ymm5[8],ymm3[9],ymm5[10,11],ymm3[12],ymm5[13,14],ymm3[15]469; AVX2-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,0,0,0,1,1,0,2]470; AVX2-NEXT: vpermd %ymm2, %ymm4, %ymm4471; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm5 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]472; AVX2-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm3473; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]474; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,u,u,10,11,8,9,u,u,14,15,12,13,u,u]475; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]476; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]477; AVX2-NEXT: vmovdqa %xmm0, 32(%rcx)478; AVX2-NEXT: vmovdqa %ymm3, (%rcx)479; AVX2-NEXT: vzeroupper480; AVX2-NEXT: retq481;482; AVX2-FP-LABEL: store_i16_stride3_vf8:483; AVX2-FP: # %bb.0:484; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm0485; AVX2-FP-NEXT: vmovdqa (%rsi), %xmm1486; AVX2-FP-NEXT: vmovdqa (%rdx), %xmm2487; AVX2-FP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm3488; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,0,1,6,7,2,3,2,3,8,9,4,5,4,5,16,17,6,7,22,23,18,19,8,9,24,25,20,21,10,11]489; AVX2-FP-NEXT: vpshufb %ymm4, %ymm3, %ymm5490; AVX2-FP-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,0,1]491; AVX2-FP-NEXT: vpshufb %ymm4, %ymm3, %ymm3492; AVX2-FP-NEXT: vpblendw {{.*#+}} ymm3 = ymm5[0],ymm3[1],ymm5[2,3],ymm3[4],ymm5[5,6],ymm3[7],ymm5[8],ymm3[9],ymm5[10,11],ymm3[12],ymm5[13,14],ymm3[15]493; AVX2-FP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,0,0,0,1,1,0,2]494; AVX2-FP-NEXT: vpermd %ymm2, %ymm4, %ymm4495; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm5 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]496; AVX2-FP-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm3497; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]498; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,u,u,10,11,8,9,u,u,14,15,12,13,u,u]499; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]500; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]501; AVX2-FP-NEXT: vmovdqa %xmm0, 32(%rcx)502; AVX2-FP-NEXT: vmovdqa %ymm3, (%rcx)503; AVX2-FP-NEXT: vzeroupper504; AVX2-FP-NEXT: retq505;506; AVX2-FCP-LABEL: store_i16_stride3_vf8:507; AVX2-FCP: # %bb.0:508; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm0509; AVX2-FCP-NEXT: vmovdqa (%rsi), %xmm1510; AVX2-FCP-NEXT: vmovdqa (%rdx), %xmm2511; AVX2-FCP-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm3512; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,0,0,0,1,1,0,2]513; AVX2-FCP-NEXT: vpermd %ymm2, %ymm4, %ymm4514; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm5 = [0,4,1,5,1,5,2,6]515; AVX2-FCP-NEXT: vpermd %ymm3, %ymm5, %ymm3516; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[0,1,4,5,u,u,2,3,6,7,u,u,8,9,12,13,u,u,18,19,22,23,u,u,24,25,28,29,u,u,26,27]517; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm5 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]518; AVX2-FCP-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm3519; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]520; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,u,u,10,11,8,9,u,u,14,15,12,13,u,u]521; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]522; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]523; AVX2-FCP-NEXT: vmovdqa %xmm0, 32(%rcx)524; AVX2-FCP-NEXT: vmovdqa %ymm3, (%rcx)525; AVX2-FCP-NEXT: vzeroupper526; AVX2-FCP-NEXT: retq527;528; AVX512-LABEL: store_i16_stride3_vf8:529; AVX512: # %bb.0:530; AVX512-NEXT: vmovdqa (%rdi), %xmm0531; AVX512-NEXT: vmovdqa (%rsi), %xmm1532; AVX512-NEXT: vmovdqa (%rdx), %xmm2533; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm3534; AVX512-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,0,1,6,7,2,3,2,3,8,9,4,5,4,5,16,17,6,7,22,23,18,19,8,9,24,25,20,21,10,11]535; AVX512-NEXT: vpshufb %ymm4, %ymm3, %ymm5536; AVX512-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,0,1]537; AVX512-NEXT: vpshufb %ymm4, %ymm3, %ymm3538; AVX512-NEXT: vpblendw {{.*#+}} ymm3 = ymm5[0],ymm3[1],ymm5[2,3],ymm3[4],ymm5[5,6],ymm3[7],ymm5[8],ymm3[9],ymm5[10,11],ymm3[12],ymm5[13,14],ymm3[15]539; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,0,0,0,1,1,0,2]540; AVX512-NEXT: vpermd %ymm2, %ymm4, %ymm4541; AVX512-NEXT: vpternlogq {{.*#+}} ymm4 = ymm4 ^ (mem & (ymm4 ^ ymm3))542; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]543; AVX512-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,u,u,10,11,8,9,u,u,14,15,12,13,u,u]544; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]545; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]546; AVX512-NEXT: vinserti32x4 $2, %xmm0, %zmm4, %zmm1547; AVX512-NEXT: vmovdqa %xmm0, 32(%rcx)548; AVX512-NEXT: vmovdqa %ymm1, (%rcx)549; AVX512-NEXT: vzeroupper550; AVX512-NEXT: retq551;552; AVX512-FCP-LABEL: store_i16_stride3_vf8:553; AVX512-FCP: # %bb.0:554; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm0555; AVX512-FCP-NEXT: vmovdqa (%rsi), %xmm1556; AVX512-FCP-NEXT: vmovdqa (%rdx), %xmm2557; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,8,1,9,1,9,2,10]558; AVX512-FCP-NEXT: vpermi2d %ymm1, %ymm0, %ymm3559; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[0,1,4,5],zero,zero,ymm3[2,3,6,7],zero,zero,ymm3[8,9,12,13],zero,zero,ymm3[18,19,22,23],zero,zero,ymm3[24,25,28,29],zero,zero,ymm3[26,27]560; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,0,0,0,1,1,0,2]561; AVX512-FCP-NEXT: vpermd %ymm2, %ymm4, %ymm4562; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm4 = (ymm4 & ~mem) | ymm3563; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]564; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,u,u,10,11,8,9,u,u,14,15,12,13,u,u]565; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]566; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]567; AVX512-FCP-NEXT: vinserti32x4 $2, %xmm0, %zmm4, %zmm1568; AVX512-FCP-NEXT: vmovdqa %xmm0, 32(%rcx)569; AVX512-FCP-NEXT: vmovdqa %ymm1, (%rcx)570; AVX512-FCP-NEXT: vzeroupper571; AVX512-FCP-NEXT: retq572;573; AVX512DQ-LABEL: store_i16_stride3_vf8:574; AVX512DQ: # %bb.0:575; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0576; AVX512DQ-NEXT: vmovdqa (%rsi), %xmm1577; AVX512DQ-NEXT: vmovdqa (%rdx), %xmm2578; AVX512DQ-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm3579; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,0,1,6,7,2,3,2,3,8,9,4,5,4,5,16,17,6,7,22,23,18,19,8,9,24,25,20,21,10,11]580; AVX512DQ-NEXT: vpshufb %ymm4, %ymm3, %ymm5581; AVX512DQ-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,0,1]582; AVX512DQ-NEXT: vpshufb %ymm4, %ymm3, %ymm3583; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm3 = ymm5[0],ymm3[1],ymm5[2,3],ymm3[4],ymm5[5,6],ymm3[7],ymm5[8],ymm3[9],ymm5[10,11],ymm3[12],ymm5[13,14],ymm3[15]584; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,0,0,0,1,1,0,2]585; AVX512DQ-NEXT: vpermd %ymm2, %ymm4, %ymm4586; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm4 = ymm4 ^ (mem & (ymm4 ^ ymm3))587; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]588; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,u,u,10,11,8,9,u,u,14,15,12,13,u,u]589; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]590; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]591; AVX512DQ-NEXT: vinserti32x4 $2, %xmm0, %zmm4, %zmm1592; AVX512DQ-NEXT: vmovdqa %xmm0, 32(%rcx)593; AVX512DQ-NEXT: vmovdqa %ymm1, (%rcx)594; AVX512DQ-NEXT: vzeroupper595; AVX512DQ-NEXT: retq596;597; AVX512DQ-FCP-LABEL: store_i16_stride3_vf8:598; AVX512DQ-FCP: # %bb.0:599; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm0600; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %xmm1601; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %xmm2602; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,8,1,9,1,9,2,10]603; AVX512DQ-FCP-NEXT: vpermi2d %ymm1, %ymm0, %ymm3604; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[0,1,4,5],zero,zero,ymm3[2,3,6,7],zero,zero,ymm3[8,9,12,13],zero,zero,ymm3[18,19,22,23],zero,zero,ymm3[24,25,28,29],zero,zero,ymm3[26,27]605; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,0,0,0,1,1,0,2]606; AVX512DQ-FCP-NEXT: vpermd %ymm2, %ymm4, %ymm4607; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm4 = (ymm4 & ~mem) | ymm3608; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]609; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5,u,u,10,11,8,9,u,u,14,15,12,13,u,u]610; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]611; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4],xmm0[5,6],xmm1[7]612; AVX512DQ-FCP-NEXT: vinserti32x4 $2, %xmm0, %zmm4, %zmm1613; AVX512DQ-FCP-NEXT: vmovdqa %xmm0, 32(%rcx)614; AVX512DQ-FCP-NEXT: vmovdqa %ymm1, (%rcx)615; AVX512DQ-FCP-NEXT: vzeroupper616; AVX512DQ-FCP-NEXT: retq617;618; AVX512BW-LABEL: store_i16_stride3_vf8:619; AVX512BW: # %bb.0:620; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0621; AVX512BW-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0622; AVX512BW-NEXT: vinserti32x4 $2, (%rdx), %zmm0, %zmm0623; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,8,16,1,9,17,2,10,18,3,11,19,4,12,20,5,13,21,6,14,22,7,15,23,0,0,0,0,0,0,0,0]624; AVX512BW-NEXT: vpermw %zmm0, %zmm1, %zmm0625; AVX512BW-NEXT: vextracti32x4 $2, %zmm0, 32(%rcx)626; AVX512BW-NEXT: vmovdqa %ymm0, (%rcx)627; AVX512BW-NEXT: vzeroupper628; AVX512BW-NEXT: retq629;630; AVX512BW-FCP-LABEL: store_i16_stride3_vf8:631; AVX512BW-FCP: # %bb.0:632; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %xmm0633; AVX512BW-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0634; AVX512BW-FCP-NEXT: vinserti32x4 $2, (%rdx), %zmm0, %zmm0635; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,8,16,1,9,17,2,10,18,3,11,19,4,12,20,5,13,21,6,14,22,7,15,23,0,0,0,0,0,0,0,0]636; AVX512BW-FCP-NEXT: vpermw %zmm0, %zmm1, %zmm0637; AVX512BW-FCP-NEXT: vextracti32x4 $2, %zmm0, 32(%rcx)638; AVX512BW-FCP-NEXT: vmovdqa %ymm0, (%rcx)639; AVX512BW-FCP-NEXT: vzeroupper640; AVX512BW-FCP-NEXT: retq641;642; AVX512DQ-BW-LABEL: store_i16_stride3_vf8:643; AVX512DQ-BW: # %bb.0:644; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %xmm0645; AVX512DQ-BW-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0646; AVX512DQ-BW-NEXT: vinserti32x4 $2, (%rdx), %zmm0, %zmm0647; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,8,16,1,9,17,2,10,18,3,11,19,4,12,20,5,13,21,6,14,22,7,15,23,0,0,0,0,0,0,0,0]648; AVX512DQ-BW-NEXT: vpermw %zmm0, %zmm1, %zmm0649; AVX512DQ-BW-NEXT: vextracti32x4 $2, %zmm0, 32(%rcx)650; AVX512DQ-BW-NEXT: vmovdqa %ymm0, (%rcx)651; AVX512DQ-BW-NEXT: vzeroupper652; AVX512DQ-BW-NEXT: retq653;654; AVX512DQ-BW-FCP-LABEL: store_i16_stride3_vf8:655; AVX512DQ-BW-FCP: # %bb.0:656; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %xmm0657; AVX512DQ-BW-FCP-NEXT: vinserti128 $1, (%rsi), %ymm0, %ymm0658; AVX512DQ-BW-FCP-NEXT: vinserti32x4 $2, (%rdx), %zmm0, %zmm0659; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm1 = [0,8,16,1,9,17,2,10,18,3,11,19,4,12,20,5,13,21,6,14,22,7,15,23,0,0,0,0,0,0,0,0]660; AVX512DQ-BW-FCP-NEXT: vpermw %zmm0, %zmm1, %zmm0661; AVX512DQ-BW-FCP-NEXT: vextracti32x4 $2, %zmm0, 32(%rcx)662; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm0, (%rcx)663; AVX512DQ-BW-FCP-NEXT: vzeroupper664; AVX512DQ-BW-FCP-NEXT: retq665 %in.vec0 = load <8 x i16>, ptr %in.vecptr0, align 64666 %in.vec1 = load <8 x i16>, ptr %in.vecptr1, align 64667 %in.vec2 = load <8 x i16>, ptr %in.vecptr2, align 64668 %1 = shufflevector <8 x i16> %in.vec0, <8 x i16> %in.vec1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>669 %2 = shufflevector <8 x i16> %in.vec2, <8 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>670 %3 = shufflevector <16 x i16> %1, <16 x i16> %2, <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>671 %interleaved.vec = shufflevector <24 x i16> %3, <24 x i16> poison, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>672 store <24 x i16> %interleaved.vec, ptr %out.vec, align 64673 ret void674}675 676define void @store_i16_stride3_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %out.vec) nounwind {677; SSE-LABEL: store_i16_stride3_vf16:678; SSE: # %bb.0:679; SSE-NEXT: movdqa (%rdi), %xmm1680; SSE-NEXT: movdqa 16(%rdi), %xmm6681; SSE-NEXT: movdqa (%rsi), %xmm2682; SSE-NEXT: movdqa 16(%rsi), %xmm7683; SSE-NEXT: movdqa (%rdx), %xmm4684; SSE-NEXT: movdqa 16(%rdx), %xmm9685; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm6[1,1,2,2]686; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,0,65535,65535,0,65535,65535]687; SSE-NEXT: pand %xmm0, %xmm3688; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm7[3,3,3,3,4,5,6,7]689; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,4,4,4]690; SSE-NEXT: movdqa %xmm0, %xmm8691; SSE-NEXT: pandn %xmm5, %xmm8692; SSE-NEXT: por %xmm3, %xmm8693; SSE-NEXT: movdqa {{.*#+}} xmm3 = [0,65535,65535,0,65535,65535,0,65535]694; SSE-NEXT: pand %xmm3, %xmm8695; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm9[1,1,2,2]696; SSE-NEXT: movdqa %xmm3, %xmm5697; SSE-NEXT: pandn %xmm10, %xmm5698; SSE-NEXT: por %xmm8, %xmm5699; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm9[0,0,0,0]700; SSE-NEXT: movdqa %xmm0, %xmm10701; SSE-NEXT: pandn %xmm8, %xmm10702; SSE-NEXT: movdqa %xmm6, %xmm8703; SSE-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]704; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,1,2,1]705; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[0,1,2,2,4,5,6,7]706; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,7,5,4,5]707; SSE-NEXT: pand %xmm0, %xmm8708; SSE-NEXT: por %xmm10, %xmm8709; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm1[1,1,2,2]710; SSE-NEXT: pand %xmm0, %xmm10711; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm2[3,3,3,3,4,5,6,7]712; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,4,4,4]713; SSE-NEXT: movdqa %xmm0, %xmm12714; SSE-NEXT: pandn %xmm11, %xmm12715; SSE-NEXT: por %xmm10, %xmm12716; SSE-NEXT: pand %xmm3, %xmm12717; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm4[1,1,2,2]718; SSE-NEXT: pandn %xmm10, %xmm3719; SSE-NEXT: por %xmm12, %xmm3720; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm4[0,0,0,0]721; SSE-NEXT: movdqa %xmm1, %xmm11722; SSE-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm2[0],xmm11[1],xmm2[1],xmm11[2],xmm2[2],xmm11[3],xmm2[3]723; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm11[0,1,2,1]724; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[0,1,2,2,4,5,6,7]725; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,7,5,4,5]726; SSE-NEXT: pand %xmm0, %xmm11727; SSE-NEXT: pandn %xmm10, %xmm0728; SSE-NEXT: por %xmm11, %xmm0729; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]730; SSE-NEXT: movdqa {{.*#+}} xmm10 = [65535,0,65535,65535,0,65535,65535,0]731; SSE-NEXT: movdqa %xmm10, %xmm11732; SSE-NEXT: pandn %xmm9, %xmm11733; SSE-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]734; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm7[2,1,3,3]735; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[2,1,1,0,4,5,6,7]736; SSE-NEXT: pand %xmm10, %xmm6737; SSE-NEXT: por %xmm11, %xmm6738; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]739; SSE-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]740; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3]741; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,1,0,4,5,6,7]742; SSE-NEXT: pand %xmm10, %xmm1743; SSE-NEXT: pandn %xmm4, %xmm10744; SSE-NEXT: por %xmm1, %xmm10745; SSE-NEXT: movdqa %xmm10, 32(%rcx)746; SSE-NEXT: movdqa %xmm6, 80(%rcx)747; SSE-NEXT: movdqa %xmm0, (%rcx)748; SSE-NEXT: movdqa %xmm3, 16(%rcx)749; SSE-NEXT: movdqa %xmm8, 48(%rcx)750; SSE-NEXT: movdqa %xmm5, 64(%rcx)751; SSE-NEXT: retq752;753; AVX-LABEL: store_i16_stride3_vf16:754; AVX: # %bb.0:755; AVX-NEXT: vmovdqa (%rdi), %xmm0756; AVX-NEXT: vmovdqa 16(%rdi), %xmm1757; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,2]758; AVX-NEXT: vmovdqa (%rsi), %xmm3759; AVX-NEXT: vmovdqa 16(%rsi), %xmm4760; AVX-NEXT: vpshuflw {{.*#+}} xmm5 = xmm4[3,3,3,3,4,5,6,7]761; AVX-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,4,4,4]762; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm5[2],xmm2[3,4],xmm5[5],xmm2[6,7]763; AVX-NEXT: vmovdqa (%rdx), %xmm5764; AVX-NEXT: vmovdqa 16(%rdx), %xmm6765; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[1,1,2,2]766; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm7[0],xmm2[1,2],xmm7[3],xmm2[4,5],xmm7[6],xmm2[7]767; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[1,1,2,2]768; AVX-NEXT: vpshuflw {{.*#+}} xmm8 = xmm3[3,3,3,3,4,5,6,7]769; AVX-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,4,4,4]770; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm8[2],xmm7[3,4],xmm8[5],xmm7[6,7]771; AVX-NEXT: vpshufd {{.*#+}} xmm8 = xmm5[1,1,2,2]772; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm8[0],xmm7[1,2],xmm8[3],xmm7[4,5],xmm8[6],xmm7[7]773; AVX-NEXT: vpunpckhwd {{.*#+}} xmm8 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]774; AVX-NEXT: vmovdqa {{.*#+}} xmm9 = [4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]775; AVX-NEXT: vpshufb %xmm9, %xmm8, %xmm8776; AVX-NEXT: vpshufd {{.*#+}} xmm10 = xmm6[2,2,3,3]777; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm10[1],xmm8[2,3],xmm10[4],xmm8[5,6],xmm10[7]778; AVX-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]779; AVX-NEXT: vmovdqa {{.*#+}} xmm11 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]780; AVX-NEXT: vpshufb %xmm11, %xmm10, %xmm10781; AVX-NEXT: vpshufd {{.*#+}} xmm12 = xmm5[0,0,0,0]782; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1],xmm12[2],xmm10[3,4],xmm12[5],xmm10[6,7]783; AVX-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]784; AVX-NEXT: vpshufb %xmm9, %xmm0, %xmm0785; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[2,2,3,3]786; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2,3],xmm3[4],xmm0[5,6],xmm3[7]787; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]788; AVX-NEXT: vpshufb %xmm11, %xmm1, %xmm1789; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm6[0,0,0,0]790; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2],xmm1[3,4],xmm3[5],xmm1[6,7]791; AVX-NEXT: vmovdqa %xmm1, 48(%rcx)792; AVX-NEXT: vmovdqa %xmm0, 32(%rcx)793; AVX-NEXT: vmovdqa %xmm10, (%rcx)794; AVX-NEXT: vmovdqa %xmm8, 80(%rcx)795; AVX-NEXT: vmovdqa %xmm7, 16(%rcx)796; AVX-NEXT: vmovdqa %xmm2, 64(%rcx)797; AVX-NEXT: retq798;799; AVX2-LABEL: store_i16_stride3_vf16:800; AVX2: # %bb.0:801; AVX2-NEXT: vmovdqa (%rsi), %ymm1802; AVX2-NEXT: vmovdqa (%rdx), %ymm0803; AVX2-NEXT: vmovdqa (%rdi), %xmm2804; AVX2-NEXT: vmovdqa 16(%rdi), %xmm3805; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[1,1,2,2]806; AVX2-NEXT: vmovdqa (%rsi), %xmm5807; AVX2-NEXT: vmovdqa 16(%rsi), %xmm6808; AVX2-NEXT: vpshuflw {{.*#+}} xmm7 = xmm6[3,3,3,3,4,5,6,7]809; AVX2-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,4,4,4]810; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]811; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]812; AVX2-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]813; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm4, %ymm3814; AVX2-NEXT: vpmovsxbd {{.*#+}} ymm4 = [5,5,0,6,6,0,7,7]815; AVX2-NEXT: vpermd %ymm0, %ymm4, %ymm4816; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm6 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]817; AVX2-NEXT: vpblendvb %ymm6, %ymm3, %ymm4, %ymm3818; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,2,2]819; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm5[3,3,3,3,4,5,6,7]820; AVX2-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,4,4]821; AVX2-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm6[2],xmm4[3,4],xmm6[5],xmm4[6,7]822; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]823; AVX2-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]824; AVX2-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2825; AVX2-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,0,0,0,1,1,0,2]826; AVX2-NEXT: vpermd %ymm0, %ymm4, %ymm4827; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm5 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]828; AVX2-NEXT: vpblendvb %ymm5, %ymm2, %ymm4, %ymm2829; AVX2-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,3,3,0,4,4,0,5]830; AVX2-NEXT: vpermd (%rdi), %ymm4, %ymm4831; AVX2-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[10,11,0,1,12,13,12,13,2,3,2,3,14,15,4,5,26,27,16,17,28,29,28,29,18,19,18,19,30,31,20,21]832; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm5 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]833; AVX2-NEXT: vpblendvb %ymm5, %ymm1, %ymm4, %ymm1834; AVX2-NEXT: vpmovsxbd {{.*#+}} ymm4 = [2,0,3,3,0,4,4,0]835; AVX2-NEXT: vpermd %ymm0, %ymm4, %ymm0836; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm4 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]837; AVX2-NEXT: vpblendvb %ymm4, %ymm1, %ymm0, %ymm0838; AVX2-NEXT: vmovdqa %ymm0, 32(%rcx)839; AVX2-NEXT: vmovdqa %ymm2, (%rcx)840; AVX2-NEXT: vmovdqa %ymm3, 64(%rcx)841; AVX2-NEXT: vzeroupper842; AVX2-NEXT: retq843;844; AVX2-FP-LABEL: store_i16_stride3_vf16:845; AVX2-FP: # %bb.0:846; AVX2-FP-NEXT: vmovdqa (%rsi), %ymm1847; AVX2-FP-NEXT: vmovdqa (%rdx), %ymm0848; AVX2-FP-NEXT: vmovdqa (%rsi), %xmm2849; AVX2-FP-NEXT: vmovdqa 16(%rsi), %xmm3850; AVX2-FP-NEXT: vpbroadcastd {{.*#+}} xmm4 = [6,7,8,9,6,7,8,9,6,7,8,9,6,7,8,9]851; AVX2-FP-NEXT: vpshufb %xmm4, %xmm2, %xmm5852; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm6853; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm7854; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,2,2]855; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm5 = xmm8[0,1],xmm5[2],xmm8[3,4],xmm5[5],xmm8[6,7]856; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]857; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]858; AVX2-FP-NEXT: vinserti128 $1, %xmm5, %ymm2, %ymm2859; AVX2-FP-NEXT: vpmovsxbd {{.*#+}} ymm5 = [0,0,0,0,1,1,0,2]860; AVX2-FP-NEXT: vpermd %ymm0, %ymm5, %ymm5861; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm6 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]862; AVX2-FP-NEXT: vpblendvb %ymm6, %ymm2, %ymm5, %ymm2863; AVX2-FP-NEXT: vpshufb %xmm4, %xmm3, %xmm4864; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm5 = xmm7[1,1,2,2]865; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2],xmm5[3,4],xmm4[5],xmm5[6,7]866; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm7[4],xmm3[5],xmm7[5],xmm3[6],xmm7[6],xmm3[7],xmm7[7]867; AVX2-FP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]868; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm4, %ymm3869; AVX2-FP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [5,5,0,6,6,0,7,7]870; AVX2-FP-NEXT: vpermd %ymm0, %ymm4, %ymm4871; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm5 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]872; AVX2-FP-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm3873; AVX2-FP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,3,3,0,4,4,0,5]874; AVX2-FP-NEXT: vpermd (%rdi), %ymm4, %ymm4875; AVX2-FP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[10,11,0,1,12,13,12,13,2,3,2,3,14,15,4,5,26,27,16,17,28,29,28,29,18,19,18,19,30,31,20,21]876; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm5 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]877; AVX2-FP-NEXT: vpblendvb %ymm5, %ymm1, %ymm4, %ymm1878; AVX2-FP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [2,0,3,3,0,4,4,0]879; AVX2-FP-NEXT: vpermd %ymm0, %ymm4, %ymm0880; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm4 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]881; AVX2-FP-NEXT: vpblendvb %ymm4, %ymm1, %ymm0, %ymm0882; AVX2-FP-NEXT: vmovdqa %ymm0, 32(%rcx)883; AVX2-FP-NEXT: vmovdqa %ymm3, 64(%rcx)884; AVX2-FP-NEXT: vmovdqa %ymm2, (%rcx)885; AVX2-FP-NEXT: vzeroupper886; AVX2-FP-NEXT: retq887;888; AVX2-FCP-LABEL: store_i16_stride3_vf16:889; AVX2-FCP: # %bb.0:890; AVX2-FCP-NEXT: vmovdqa (%rsi), %ymm1891; AVX2-FCP-NEXT: vmovdqa (%rdx), %ymm0892; AVX2-FCP-NEXT: vmovdqa (%rsi), %xmm2893; AVX2-FCP-NEXT: vmovdqa 16(%rsi), %xmm3894; AVX2-FCP-NEXT: vpbroadcastd {{.*#+}} xmm4 = [6,7,8,9,6,7,8,9,6,7,8,9,6,7,8,9]895; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm2, %xmm5896; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm6897; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm7898; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,2,2]899; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm8[0,1],xmm5[2],xmm8[3,4],xmm5[5],xmm8[6,7]900; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]901; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]902; AVX2-FCP-NEXT: vinserti128 $1, %xmm5, %ymm2, %ymm2903; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm5 = [0,0,0,0,1,1,0,2]904; AVX2-FCP-NEXT: vpermd %ymm0, %ymm5, %ymm5905; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm6 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]906; AVX2-FCP-NEXT: vpblendvb %ymm6, %ymm2, %ymm5, %ymm2907; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm3, %xmm4908; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm5 = xmm7[1,1,2,2]909; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2],xmm5[3,4],xmm4[5],xmm5[6,7]910; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm7[4],xmm3[5],xmm7[5],xmm3[6],xmm7[6],xmm3[7],xmm7[7]911; AVX2-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]912; AVX2-FCP-NEXT: vinserti128 $1, %xmm3, %ymm4, %ymm3913; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [5,5,0,6,6,0,7,7]914; AVX2-FCP-NEXT: vpermd %ymm0, %ymm4, %ymm4915; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm5 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]916; AVX2-FCP-NEXT: vpblendvb %ymm5, %ymm3, %ymm4, %ymm3917; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,3,3,0,4,4,0,5]918; AVX2-FCP-NEXT: vpermd (%rdi), %ymm4, %ymm4919; AVX2-FCP-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[10,11,0,1,12,13,12,13,2,3,2,3,14,15,4,5,26,27,16,17,28,29,28,29,18,19,18,19,30,31,20,21]920; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm5 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]921; AVX2-FCP-NEXT: vpblendvb %ymm5, %ymm1, %ymm4, %ymm1922; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [2,0,3,3,0,4,4,0]923; AVX2-FCP-NEXT: vpermd %ymm0, %ymm4, %ymm0924; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm4 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]925; AVX2-FCP-NEXT: vpblendvb %ymm4, %ymm1, %ymm0, %ymm0926; AVX2-FCP-NEXT: vmovdqa %ymm0, 32(%rcx)927; AVX2-FCP-NEXT: vmovdqa %ymm3, 64(%rcx)928; AVX2-FCP-NEXT: vmovdqa %ymm2, (%rcx)929; AVX2-FCP-NEXT: vzeroupper930; AVX2-FCP-NEXT: retq931;932; AVX512-LABEL: store_i16_stride3_vf16:933; AVX512: # %bb.0:934; AVX512-NEXT: vmovdqa (%rdi), %ymm1935; AVX512-NEXT: vmovdqa (%rsi), %ymm2936; AVX512-NEXT: vmovdqa (%rdx), %ymm0937; AVX512-NEXT: vmovdqa (%rsi), %xmm3938; AVX512-NEXT: vmovdqa 16(%rsi), %xmm4939; AVX512-NEXT: vprold $16, %xmm3, %xmm5940; AVX512-NEXT: vmovdqa (%rdi), %xmm6941; AVX512-NEXT: vmovdqa 16(%rdi), %xmm7942; AVX512-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,2,2]943; AVX512-NEXT: vpblendw {{.*#+}} xmm5 = xmm8[0,1],xmm5[2],xmm8[3,4],xmm5[5],xmm8[6,7]944; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]945; AVX512-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,2,3,u,u,4,5,6,7,u,u,8,9,10,11]946; AVX512-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3947; AVX512-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,ymm1[u,u,12,13],zero,zero,ymm1[u,u,14,15],zero,zero,ymm1[u,u,16,17],zero,zero,ymm1[u,u,18,19],zero,zero,ymm1[u,u,20,21],zero,zero948; AVX512-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[10,11,u,u],zero,zero,ymm2[12,13,u,u],zero,zero,ymm2[14,15,u,u],zero,zero,ymm2[16,17,u,u],zero,zero,ymm2[18,19,u,u],zero,zero,ymm2[20,21]949; AVX512-NEXT: vpor %ymm1, %ymm2, %ymm1950; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1951; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,0,0,0,1,1,0,2]952; AVX512-NEXT: vpermd %ymm0, %ymm2, %ymm2953; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2954; AVX512-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,ymm0[10,11],zero,zero,zero,zero,ymm0[12,13],zero,zero,zero,zero,ymm0[14,15],zero,zero,zero,zero,ymm0[16,17],zero,zero,zero,zero,ymm0[18,19],zero,zero,zero,zero955; AVX512-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2956; AVX512-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 | (zmm1 & mem)957; AVX512-NEXT: vprold $16, %xmm4, %xmm1958; AVX512-NEXT: vpshufd {{.*#+}} xmm3 = xmm7[1,1,2,2]959; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1],xmm1[2],xmm3[3,4],xmm1[5],xmm3[6,7]960; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm7[4],xmm4[5],xmm7[5],xmm4[6],xmm7[6],xmm4[7],xmm7[7]961; AVX512-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[4,5,u,u,10,11,8,9,u,u,14,15,12,13,u,u]962; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1963; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm3 = [5,5,0,6,6,0,7,7]964; AVX512-NEXT: vpermd %ymm0, %ymm3, %ymm0965; AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))966; AVX512-NEXT: vmovdqa %ymm0, 64(%rcx)967; AVX512-NEXT: vmovdqa64 %zmm2, (%rcx)968; AVX512-NEXT: vzeroupper969; AVX512-NEXT: retq970;971; AVX512-FCP-LABEL: store_i16_stride3_vf16:972; AVX512-FCP: # %bb.0:973; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm1974; AVX512-FCP-NEXT: vmovdqa (%rsi), %ymm2975; AVX512-FCP-NEXT: vmovdqa (%rdx), %ymm0976; AVX512-FCP-NEXT: vmovdqa (%rsi), %xmm3977; AVX512-FCP-NEXT: vmovdqa 16(%rsi), %xmm4978; AVX512-FCP-NEXT: vprold $16, %xmm3, %xmm5979; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm6980; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm7981; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,2,2]982; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm8[0,1],xmm5[2],xmm8[3,4],xmm5[5],xmm8[6,7]983; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]984; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,2,3,u,u,4,5,6,7,u,u,8,9,10,11]985; AVX512-FCP-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm3986; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,ymm1[u,u,12,13],zero,zero,ymm1[u,u,14,15],zero,zero,ymm1[u,u,16,17],zero,zero,ymm1[u,u,18,19],zero,zero,ymm1[u,u,20,21],zero,zero987; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[10,11,u,u],zero,zero,ymm2[12,13,u,u],zero,zero,ymm2[14,15,u,u],zero,zero,ymm2[16,17,u,u],zero,zero,ymm2[18,19,u,u],zero,zero,ymm2[20,21]988; AVX512-FCP-NEXT: vpor %ymm1, %ymm2, %ymm1989; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1990; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,0,0,0,1,1,0,2]991; AVX512-FCP-NEXT: vpermd %ymm0, %ymm2, %ymm2992; AVX512-FCP-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2993; AVX512-FCP-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,ymm0[10,11],zero,zero,zero,zero,ymm0[12,13],zero,zero,zero,zero,ymm0[14,15],zero,zero,zero,zero,ymm0[16,17],zero,zero,zero,zero,ymm0[18,19],zero,zero,zero,zero994; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2995; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 | (zmm1 & mem)996; AVX512-FCP-NEXT: vprold $16, %xmm4, %xmm1997; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm3 = xmm7[1,1,2,2]998; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1],xmm1[2],xmm3[3,4],xmm1[5],xmm3[6,7]999; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm7[4],xmm4[5],xmm7[5],xmm4[6],xmm7[6],xmm4[7],xmm7[7]1000; AVX512-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[4,5,u,u,10,11,8,9,u,u,14,15,12,13,u,u]1001; AVX512-FCP-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm11002; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [5,5,0,6,6,0,7,7]1003; AVX512-FCP-NEXT: vpermd %ymm0, %ymm3, %ymm01004; AVX512-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))1005; AVX512-FCP-NEXT: vmovdqa %ymm0, 64(%rcx)1006; AVX512-FCP-NEXT: vmovdqa64 %zmm2, (%rcx)1007; AVX512-FCP-NEXT: vzeroupper1008; AVX512-FCP-NEXT: retq1009;1010; AVX512DQ-LABEL: store_i16_stride3_vf16:1011; AVX512DQ: # %bb.0:1012; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm11013; AVX512DQ-NEXT: vmovdqa (%rsi), %ymm21014; AVX512DQ-NEXT: vmovdqa (%rdx), %ymm01015; AVX512DQ-NEXT: vmovdqa (%rsi), %xmm31016; AVX512DQ-NEXT: vmovdqa 16(%rsi), %xmm41017; AVX512DQ-NEXT: vprold $16, %xmm3, %xmm51018; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm61019; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm71020; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,2,2]1021; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm5 = xmm8[0,1],xmm5[2],xmm8[3,4],xmm5[5],xmm8[6,7]1022; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]1023; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,2,3,u,u,4,5,6,7,u,u,8,9,10,11]1024; AVX512DQ-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm31025; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,ymm1[u,u,12,13],zero,zero,ymm1[u,u,14,15],zero,zero,ymm1[u,u,16,17],zero,zero,ymm1[u,u,18,19],zero,zero,ymm1[u,u,20,21],zero,zero1026; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[10,11,u,u],zero,zero,ymm2[12,13,u,u],zero,zero,ymm2[14,15,u,u],zero,zero,ymm2[16,17,u,u],zero,zero,ymm2[18,19,u,u],zero,zero,ymm2[20,21]1027; AVX512DQ-NEXT: vpor %ymm1, %ymm2, %ymm11028; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm11029; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,0,0,0,1,1,0,2]1030; AVX512DQ-NEXT: vpermd %ymm0, %ymm2, %ymm21031; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm21032; AVX512DQ-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,ymm0[10,11],zero,zero,zero,zero,ymm0[12,13],zero,zero,zero,zero,ymm0[14,15],zero,zero,zero,zero,ymm0[16,17],zero,zero,zero,zero,ymm0[18,19],zero,zero,zero,zero1033; AVX512DQ-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm21034; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 | (zmm1 & mem)1035; AVX512DQ-NEXT: vprold $16, %xmm4, %xmm11036; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm3 = xmm7[1,1,2,2]1037; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1],xmm1[2],xmm3[3,4],xmm1[5],xmm3[6,7]1038; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm7[4],xmm4[5],xmm7[5],xmm4[6],xmm7[6],xmm4[7],xmm7[7]1039; AVX512DQ-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[4,5,u,u,10,11,8,9,u,u,14,15,12,13,u,u]1040; AVX512DQ-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm11041; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm3 = [5,5,0,6,6,0,7,7]1042; AVX512DQ-NEXT: vpermd %ymm0, %ymm3, %ymm01043; AVX512DQ-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))1044; AVX512DQ-NEXT: vmovdqa %ymm0, 64(%rcx)1045; AVX512DQ-NEXT: vmovdqa64 %zmm2, (%rcx)1046; AVX512DQ-NEXT: vzeroupper1047; AVX512DQ-NEXT: retq1048;1049; AVX512DQ-FCP-LABEL: store_i16_stride3_vf16:1050; AVX512DQ-FCP: # %bb.0:1051; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm11052; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %ymm21053; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %ymm01054; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %xmm31055; AVX512DQ-FCP-NEXT: vmovdqa 16(%rsi), %xmm41056; AVX512DQ-FCP-NEXT: vprold $16, %xmm3, %xmm51057; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm61058; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm71059; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[1,1,2,2]1060; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm8[0,1],xmm5[2],xmm8[3,4],xmm5[5],xmm8[6,7]1061; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]1062; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,2,3,u,u,4,5,6,7,u,u,8,9,10,11]1063; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm5, %ymm3, %ymm31064; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm1 = zero,zero,ymm1[u,u,12,13],zero,zero,ymm1[u,u,14,15],zero,zero,ymm1[u,u,16,17],zero,zero,ymm1[u,u,18,19],zero,zero,ymm1[u,u,20,21],zero,zero1065; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[10,11,u,u],zero,zero,ymm2[12,13,u,u],zero,zero,ymm2[14,15,u,u],zero,zero,ymm2[16,17,u,u],zero,zero,ymm2[18,19,u,u],zero,zero,ymm2[20,21]1066; AVX512DQ-FCP-NEXT: vpor %ymm1, %ymm2, %ymm11067; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm11068; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,0,0,0,1,1,0,2]1069; AVX512DQ-FCP-NEXT: vpermd %ymm0, %ymm2, %ymm21070; AVX512DQ-FCP-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm21071; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,ymm0[10,11],zero,zero,zero,zero,ymm0[12,13],zero,zero,zero,zero,ymm0[14,15],zero,zero,zero,zero,ymm0[16,17],zero,zero,zero,zero,ymm0[18,19],zero,zero,zero,zero1072; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm21073; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm2 = zmm2 | (zmm1 & mem)1074; AVX512DQ-FCP-NEXT: vprold $16, %xmm4, %xmm11075; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm3 = xmm7[1,1,2,2]1076; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0,1],xmm1[2],xmm3[3,4],xmm1[5],xmm3[6,7]1077; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm7[4],xmm4[5],xmm7[5],xmm4[6],xmm7[6],xmm4[7],xmm7[7]1078; AVX512DQ-FCP-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[4,5,u,u,10,11,8,9,u,u,14,15,12,13,u,u]1079; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm11080; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm3 = [5,5,0,6,6,0,7,7]1081; AVX512DQ-FCP-NEXT: vpermd %ymm0, %ymm3, %ymm01082; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 ^ (mem & (ymm0 ^ ymm1))1083; AVX512DQ-FCP-NEXT: vmovdqa %ymm0, 64(%rcx)1084; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, (%rcx)1085; AVX512DQ-FCP-NEXT: vzeroupper1086; AVX512DQ-FCP-NEXT: retq1087;1088; AVX512BW-LABEL: store_i16_stride3_vf16:1089; AVX512BW: # %bb.0:1090; AVX512BW-NEXT: vmovdqa (%rdi), %ymm01091; AVX512BW-NEXT: vmovdqa (%rdx), %ymm11092; AVX512BW-NEXT: vinserti64x4 $1, (%rsi), %zmm0, %zmm01093; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [42,11,27,43,12,28,44,13,29,45,14,30,46,15,31,47]1094; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm21095; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,16,32,1,17,33,2,18,34,3,19,35,4,20,36,5,21,37,6,22,38,7,23,39,8,24,40,9,25,41,10,26]1096; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm31097; AVX512BW-NEXT: vmovdqa64 %zmm3, (%rcx)1098; AVX512BW-NEXT: vmovdqa %ymm2, 64(%rcx)1099; AVX512BW-NEXT: vzeroupper1100; AVX512BW-NEXT: retq1101;1102; AVX512BW-FCP-LABEL: store_i16_stride3_vf16:1103; AVX512BW-FCP: # %bb.0:1104; AVX512BW-FCP-NEXT: vmovdqa (%rdi), %ymm01105; AVX512BW-FCP-NEXT: vmovdqa (%rdx), %ymm11106; AVX512BW-FCP-NEXT: vinserti64x4 $1, (%rsi), %zmm0, %zmm01107; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [42,11,27,43,12,28,44,13,29,45,14,30,46,15,31,47]1108; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm21109; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,16,32,1,17,33,2,18,34,3,19,35,4,20,36,5,21,37,6,22,38,7,23,39,8,24,40,9,25,41,10,26]1110; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm31111; AVX512BW-FCP-NEXT: vmovdqa64 %zmm3, (%rcx)1112; AVX512BW-FCP-NEXT: vmovdqa %ymm2, 64(%rcx)1113; AVX512BW-FCP-NEXT: vzeroupper1114; AVX512BW-FCP-NEXT: retq1115;1116; AVX512DQ-BW-LABEL: store_i16_stride3_vf16:1117; AVX512DQ-BW: # %bb.0:1118; AVX512DQ-BW-NEXT: vmovdqa (%rdi), %ymm01119; AVX512DQ-BW-NEXT: vmovdqa (%rdx), %ymm11120; AVX512DQ-BW-NEXT: vinserti64x4 $1, (%rsi), %zmm0, %zmm01121; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [42,11,27,43,12,28,44,13,29,45,14,30,46,15,31,47]1122; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm21123; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,16,32,1,17,33,2,18,34,3,19,35,4,20,36,5,21,37,6,22,38,7,23,39,8,24,40,9,25,41,10,26]1124; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm31125; AVX512DQ-BW-NEXT: vmovdqa64 %zmm3, (%rcx)1126; AVX512DQ-BW-NEXT: vmovdqa %ymm2, 64(%rcx)1127; AVX512DQ-BW-NEXT: vzeroupper1128; AVX512DQ-BW-NEXT: retq1129;1130; AVX512DQ-BW-FCP-LABEL: store_i16_stride3_vf16:1131; AVX512DQ-BW-FCP: # %bb.0:1132; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdi), %ymm01133; AVX512DQ-BW-FCP-NEXT: vmovdqa (%rdx), %ymm11134; AVX512DQ-BW-FCP-NEXT: vinserti64x4 $1, (%rsi), %zmm0, %zmm01135; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} ymm2 = [42,11,27,43,12,28,44,13,29,45,14,30,46,15,31,47]1136; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm21137; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,16,32,1,17,33,2,18,34,3,19,35,4,20,36,5,21,37,6,22,38,7,23,39,8,24,40,9,25,41,10,26]1138; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm31139; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm3, (%rcx)1140; AVX512DQ-BW-FCP-NEXT: vmovdqa %ymm2, 64(%rcx)1141; AVX512DQ-BW-FCP-NEXT: vzeroupper1142; AVX512DQ-BW-FCP-NEXT: retq1143 %in.vec0 = load <16 x i16>, ptr %in.vecptr0, align 641144 %in.vec1 = load <16 x i16>, ptr %in.vecptr1, align 641145 %in.vec2 = load <16 x i16>, ptr %in.vecptr2, align 641146 %1 = shufflevector <16 x i16> %in.vec0, <16 x i16> %in.vec1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>1147 %2 = shufflevector <16 x i16> %in.vec2, <16 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>1148 %3 = shufflevector <32 x i16> %1, <32 x i16> %2, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>1149 %interleaved.vec = shufflevector <48 x i16> %3, <48 x i16> poison, <48 x i32> <i32 0, i32 16, i32 32, i32 1, i32 17, i32 33, i32 2, i32 18, i32 34, i32 3, i32 19, i32 35, i32 4, i32 20, i32 36, i32 5, i32 21, i32 37, i32 6, i32 22, i32 38, i32 7, i32 23, i32 39, i32 8, i32 24, i32 40, i32 9, i32 25, i32 41, i32 10, i32 26, i32 42, i32 11, i32 27, i32 43, i32 12, i32 28, i32 44, i32 13, i32 29, i32 45, i32 14, i32 30, i32 46, i32 15, i32 31, i32 47>1150 store <48 x i16> %interleaved.vec, ptr %out.vec, align 641151 ret void1152}1153 1154define void @store_i16_stride3_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %out.vec) nounwind {1155; SSE-LABEL: store_i16_stride3_vf32:1156; SSE: # %bb.0:1157; SSE-NEXT: movdqa 16(%rdi), %xmm61158; SSE-NEXT: movdqa 32(%rdi), %xmm41159; SSE-NEXT: movdqa 48(%rdi), %xmm91160; SSE-NEXT: movdqa 16(%rsi), %xmm71161; SSE-NEXT: movdqa 32(%rsi), %xmm81162; SSE-NEXT: movdqa 48(%rsi), %xmm111163; SSE-NEXT: movdqa 32(%rdx), %xmm101164; SSE-NEXT: movdqa 48(%rdx), %xmm121165; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1166; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm9[1,1,2,2]1167; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1168; SSE-NEXT: movdqa {{.*#+}} xmm5 = [65535,65535,0,65535,65535,0,65535,65535]1169; SSE-NEXT: pand %xmm5, %xmm11170; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm11[3,3,3,3,4,5,6,7]1171; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,4,4]1172; SSE-NEXT: movdqa %xmm5, %xmm31173; SSE-NEXT: pandn %xmm2, %xmm31174; SSE-NEXT: por %xmm1, %xmm31175; SSE-NEXT: movdqa {{.*#+}} xmm2 = [0,65535,65535,0,65535,65535,0,65535]1176; SSE-NEXT: pand %xmm2, %xmm31177; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm12[1,1,2,2]1178; SSE-NEXT: movdqa %xmm2, %xmm01179; SSE-NEXT: pandn %xmm1, %xmm01180; SSE-NEXT: por %xmm3, %xmm01181; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1182; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm12[0,0,0,0]1183; SSE-NEXT: movdqa %xmm5, %xmm31184; SSE-NEXT: pandn %xmm1, %xmm31185; SSE-NEXT: movdqa %xmm9, %xmm11186; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm11[0],xmm1[1],xmm11[1],xmm1[2],xmm11[2],xmm1[3],xmm11[3]1187; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]1188; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,2,4,5,6,7]1189; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm1[0,1,2,3,7,5,4,5]1190; SSE-NEXT: pand %xmm5, %xmm91191; SSE-NEXT: por %xmm3, %xmm91192; SSE-NEXT: movdqa %xmm4, %xmm01193; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1194; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm4[1,1,2,2]1195; SSE-NEXT: pand %xmm5, %xmm11196; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm8[3,3,3,3,4,5,6,7]1197; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,4,4]1198; SSE-NEXT: movdqa %xmm5, %xmm41199; SSE-NEXT: pandn %xmm3, %xmm41200; SSE-NEXT: por %xmm1, %xmm41201; SSE-NEXT: pand %xmm2, %xmm41202; SSE-NEXT: movdqa %xmm10, %xmm31203; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1204; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm10[1,1,2,2]1205; SSE-NEXT: movdqa %xmm2, %xmm101206; SSE-NEXT: pandn %xmm1, %xmm101207; SSE-NEXT: por %xmm4, %xmm101208; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,0,0,0]1209; SSE-NEXT: movdqa %xmm5, %xmm31210; SSE-NEXT: pandn %xmm1, %xmm31211; SSE-NEXT: movdqa %xmm0, %xmm11212; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3]1213; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]1214; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,2,4,5,6,7]1215; SSE-NEXT: pshufhw {{.*#+}} xmm13 = xmm1[0,1,2,3,7,5,4,5]1216; SSE-NEXT: pand %xmm5, %xmm131217; SSE-NEXT: por %xmm3, %xmm131218; SSE-NEXT: movdqa %xmm7, %xmm01219; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1220; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm7[3,3,3,3,4,5,6,7]1221; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]1222; SSE-NEXT: movdqa %xmm5, %xmm31223; SSE-NEXT: pandn %xmm1, %xmm31224; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1225; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[1,1,2,2]1226; SSE-NEXT: pand %xmm5, %xmm11227; SSE-NEXT: por %xmm1, %xmm31228; SSE-NEXT: movdqa 16(%rdx), %xmm71229; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[1,1,2,2]1230; SSE-NEXT: movdqa %xmm2, %xmm141231; SSE-NEXT: pandn %xmm1, %xmm141232; SSE-NEXT: pand %xmm2, %xmm31233; SSE-NEXT: por %xmm3, %xmm141234; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[0,0,0,0]1235; SSE-NEXT: movdqa %xmm5, %xmm31236; SSE-NEXT: pandn %xmm1, %xmm31237; SSE-NEXT: movdqa %xmm6, %xmm11238; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]1239; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]1240; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,2,4,5,6,7]1241; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm1[0,1,2,3,7,5,4,5]1242; SSE-NEXT: pand %xmm5, %xmm151243; SSE-NEXT: por %xmm3, %xmm151244; SSE-NEXT: movdqa (%rsi), %xmm41245; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm4[3,3,3,3,4,5,6,7]1246; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,4,4]1247; SSE-NEXT: movdqa %xmm5, %xmm01248; SSE-NEXT: pandn %xmm3, %xmm01249; SSE-NEXT: movdqa (%rdi), %xmm61250; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm6[1,1,2,2]1251; SSE-NEXT: pand %xmm5, %xmm31252; SSE-NEXT: por %xmm3, %xmm01253; SSE-NEXT: pand %xmm2, %xmm01254; SSE-NEXT: movdqa (%rdx), %xmm31255; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm3[1,1,2,2]1256; SSE-NEXT: pandn %xmm12, %xmm21257; SSE-NEXT: por %xmm0, %xmm21258; SSE-NEXT: movdqa %xmm6, %xmm01259; SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]1260; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1]1261; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,2,4,5,6,7]1262; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,4,5]1263; SSE-NEXT: pand %xmm5, %xmm01264; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm3[0,0,0,0]1265; SSE-NEXT: pandn %xmm12, %xmm51266; SSE-NEXT: por %xmm0, %xmm51267; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload1268; SSE-NEXT: # xmm1 = mem[2,2,3,3]1269; SSE-NEXT: movdqa {{.*#+}} xmm12 = [65535,0,65535,65535,0,65535,65535,0]1270; SSE-NEXT: movdqa %xmm12, %xmm01271; SSE-NEXT: pandn %xmm1, %xmm01272; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload1273; SSE-NEXT: # xmm11 = xmm11[4],mem[4],xmm11[5],mem[5],xmm11[6],mem[6],xmm11[7],mem[7]1274; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm11[2,1,3,3]1275; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm1[2,1,1,0,4,5,6,7]1276; SSE-NEXT: pand %xmm12, %xmm111277; SSE-NEXT: por %xmm0, %xmm111278; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload1279; SSE-NEXT: # xmm0 = mem[2,2,3,3]1280; SSE-NEXT: movdqa %xmm12, %xmm11281; SSE-NEXT: pandn %xmm0, %xmm11282; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload1283; SSE-NEXT: # xmm8 = xmm8[4],mem[4],xmm8[5],mem[5],xmm8[6],mem[6],xmm8[7],mem[7]1284; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm8[2,1,3,3]1285; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,1,0,4,5,6,7]1286; SSE-NEXT: pand %xmm12, %xmm01287; SSE-NEXT: por %xmm1, %xmm01288; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[2,2,3,3]1289; SSE-NEXT: movdqa %xmm12, %xmm71290; SSE-NEXT: pandn %xmm1, %xmm71291; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload1292; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload1293; SSE-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]1294; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,3,3]1295; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,1,0,4,5,6,7]1296; SSE-NEXT: pand %xmm12, %xmm11297; SSE-NEXT: por %xmm7, %xmm11298; SSE-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]1299; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,2,3,3]1300; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,1,3,3]1301; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[2,1,1,0,4,5,6,7]1302; SSE-NEXT: pand %xmm12, %xmm41303; SSE-NEXT: pandn %xmm3, %xmm121304; SSE-NEXT: por %xmm4, %xmm121305; SSE-NEXT: movdqa %xmm12, 32(%rcx)1306; SSE-NEXT: movdqa %xmm1, 80(%rcx)1307; SSE-NEXT: movdqa %xmm0, 128(%rcx)1308; SSE-NEXT: movdqa %xmm11, 176(%rcx)1309; SSE-NEXT: movdqa %xmm5, (%rcx)1310; SSE-NEXT: movdqa %xmm2, 16(%rcx)1311; SSE-NEXT: movdqa %xmm15, 48(%rcx)1312; SSE-NEXT: movdqa %xmm14, 64(%rcx)1313; SSE-NEXT: movdqa %xmm13, 96(%rcx)1314; SSE-NEXT: movdqa %xmm10, 112(%rcx)1315; SSE-NEXT: movdqa %xmm9, 144(%rcx)1316; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1317; SSE-NEXT: movaps %xmm0, 160(%rcx)1318; SSE-NEXT: retq1319;1320; AVX-LABEL: store_i16_stride3_vf32:1321; AVX: # %bb.0:1322; AVX-NEXT: vmovdqa (%rdi), %xmm141323; AVX-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1324; AVX-NEXT: vmovdqa 16(%rdi), %xmm21325; AVX-NEXT: vmovdqa 32(%rdi), %xmm81326; AVX-NEXT: vmovdqa 48(%rdi), %xmm91327; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm9[1,1,2,2]1328; AVX-NEXT: vmovdqa (%rsi), %xmm31329; AVX-NEXT: vmovdqa 16(%rsi), %xmm51330; AVX-NEXT: vmovdqa 32(%rsi), %xmm101331; AVX-NEXT: vmovdqa 48(%rsi), %xmm121332; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm12[3,3,3,3,4,5,6,7]1333; AVX-NEXT: vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,4,4,4]1334; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm0[0,1],xmm4[2],xmm0[3,4],xmm4[5],xmm0[6,7]1335; AVX-NEXT: vmovdqa 16(%rdx), %xmm61336; AVX-NEXT: vmovdqa 32(%rdx), %xmm131337; AVX-NEXT: vmovdqa 48(%rdx), %xmm01338; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[1,1,2,2]1339; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm7[0],xmm4[1,2],xmm7[3],xmm4[4,5],xmm7[6],xmm4[7]1340; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1341; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm8[1,1,2,2]1342; AVX-NEXT: vpshuflw {{.*#+}} xmm7 = xmm10[3,3,3,3,4,5,6,7]1343; AVX-NEXT: vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,4,4,4]1344; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]1345; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm13[1,1,2,2]1346; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm7[0],xmm4[1,2],xmm7[3],xmm4[4,5],xmm7[6],xmm4[7]1347; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1348; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm2[1,1,2,2]1349; AVX-NEXT: vpshuflw {{.*#+}} xmm11 = xmm5[3,3,3,3,4,5,6,7]1350; AVX-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,4,4,4]1351; AVX-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm11[2],xmm7[3,4],xmm11[5],xmm7[6,7]1352; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm6[1,1,2,2]1353; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm11[0],xmm7[1,2],xmm11[3],xmm7[4,5],xmm11[6],xmm7[7]1354; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1355; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm14[1,1,2,2]1356; AVX-NEXT: vpshuflw {{.*#+}} xmm14 = xmm3[3,3,3,3,4,5,6,7]1357; AVX-NEXT: vpshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,4,4,4,4]1358; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0,1],xmm14[2],xmm11[3,4],xmm14[5],xmm11[6,7]1359; AVX-NEXT: vmovdqa (%rdx), %xmm151360; AVX-NEXT: vpshufd {{.*#+}} xmm14 = xmm15[1,1,2,2]1361; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm14[0],xmm11[1,2],xmm14[3],xmm11[4,5],xmm14[6],xmm11[7]1362; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1363; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm9[0],xmm12[0],xmm9[1],xmm12[1],xmm9[2],xmm12[2],xmm9[3],xmm12[3]1364; AVX-NEXT: vmovdqa {{.*#+}} xmm7 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]1365; AVX-NEXT: vpshufb %xmm7, %xmm14, %xmm141366; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[0,0,0,0]1367; AVX-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0,1],xmm4[2],xmm14[3,4],xmm4[5],xmm14[6,7]1368; AVX-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm10[4],xmm8[4],xmm10[5],xmm8[5],xmm10[6],xmm8[6],xmm10[7],xmm8[7]1369; AVX-NEXT: vmovdqa {{.*#+}} xmm1 = [4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]1370; AVX-NEXT: vpshufb %xmm1, %xmm4, %xmm41371; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm13[2,2,3,3]1372; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm11[1],xmm4[2,3],xmm11[4],xmm4[5,6],xmm11[7]1373; AVX-NEXT: vpunpckhwd {{.*#+}} xmm9 = xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]1374; AVX-NEXT: vpshufb %xmm1, %xmm9, %xmm91375; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]1376; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm9[0],xmm0[1],xmm9[2,3],xmm0[4],xmm9[5,6],xmm0[7]1377; AVX-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3]1378; AVX-NEXT: vpshufb %xmm7, %xmm8, %xmm81379; AVX-NEXT: vpshufd {{.*#+}} xmm9 = xmm13[0,0,0,0]1380; AVX-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2],xmm8[3,4],xmm9[5],xmm8[6,7]1381; AVX-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]1382; AVX-NEXT: vpshufb %xmm7, %xmm9, %xmm91383; AVX-NEXT: vpshufd {{.*#+}} xmm10 = xmm6[0,0,0,0]1384; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1],xmm10[2],xmm9[3,4],xmm10[5],xmm9[6,7]1385; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload1386; AVX-NEXT: vpunpckhwd {{.*#+}} xmm10 = xmm3[4],xmm12[4],xmm3[5],xmm12[5],xmm3[6],xmm12[6],xmm3[7],xmm12[7]1387; AVX-NEXT: vpshufb %xmm1, %xmm10, %xmm101388; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm15[2,2,3,3]1389; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0],xmm11[1],xmm10[2,3],xmm11[4],xmm10[5,6],xmm11[7]1390; AVX-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]1391; AVX-NEXT: vpshufb %xmm1, %xmm2, %xmm11392; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm6[2,2,3,3]1393; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3],xmm2[4],xmm1[5,6],xmm2[7]1394; AVX-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm12[0],xmm3[0],xmm12[1],xmm3[1],xmm12[2],xmm3[2],xmm12[3],xmm3[3]1395; AVX-NEXT: vpshufb %xmm7, %xmm2, %xmm21396; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm15[0,0,0,0]1397; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2],xmm2[3,4],xmm3[5],xmm2[6,7]1398; AVX-NEXT: vmovdqa %xmm2, (%rcx)1399; AVX-NEXT: vmovdqa %xmm1, 80(%rcx)1400; AVX-NEXT: vmovdqa %xmm10, 32(%rcx)1401; AVX-NEXT: vmovdqa %xmm9, 48(%rcx)1402; AVX-NEXT: vmovdqa %xmm8, 96(%rcx)1403; AVX-NEXT: vmovdqa %xmm0, 176(%rcx)1404; AVX-NEXT: vmovdqa %xmm4, 128(%rcx)1405; AVX-NEXT: vmovdqa %xmm14, 144(%rcx)1406; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1407; AVX-NEXT: vmovaps %xmm0, 16(%rcx)1408; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1409; AVX-NEXT: vmovaps %xmm0, 64(%rcx)1410; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1411; AVX-NEXT: vmovaps %xmm0, 112(%rcx)1412; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1413; AVX-NEXT: vmovaps %xmm0, 160(%rcx)1414; AVX-NEXT: retq1415;1416; AVX2-LABEL: store_i16_stride3_vf32:1417; AVX2: # %bb.0:1418; AVX2-NEXT: vmovdqa 32(%rsi), %ymm01419; AVX2-NEXT: vmovdqa (%rsi), %ymm41420; AVX2-NEXT: vmovdqa (%rdx), %ymm21421; AVX2-NEXT: vmovdqa 32(%rdx), %ymm11422; AVX2-NEXT: vmovdqa 48(%rdi), %xmm31423; AVX2-NEXT: vmovdqa 48(%rsi), %xmm51424; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]1425; AVX2-NEXT: vmovdqa {{.*#+}} xmm7 = [4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]1426; AVX2-NEXT: vpshufb %xmm7, %xmm6, %xmm61427; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,1,2,2]1428; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[3,3,3,3,4,5,6,7]1429; AVX2-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,4,4,4]1430; AVX2-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm5[2],xmm3[3,4],xmm5[5],xmm3[6,7]1431; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm3, %ymm31432; AVX2-NEXT: vpmovsxbd {{.*#+}} ymm5 = [5,5,0,6,6,0,7,7]1433; AVX2-NEXT: vpermd %ymm1, %ymm5, %ymm61434; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm8 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]1435; AVX2-NEXT: vpblendvb %ymm8, %ymm3, %ymm6, %ymm31436; AVX2-NEXT: vmovdqa (%rdi), %xmm61437; AVX2-NEXT: vmovdqa 16(%rdi), %xmm91438; AVX2-NEXT: vmovdqa 32(%rdi), %xmm101439; AVX2-NEXT: vmovdqa (%rsi), %xmm111440; AVX2-NEXT: vmovdqa 16(%rsi), %xmm121441; AVX2-NEXT: vmovdqa 32(%rsi), %xmm131442; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm14 = xmm12[4],xmm9[4],xmm12[5],xmm9[5],xmm12[6],xmm9[6],xmm12[7],xmm9[7]1443; AVX2-NEXT: vpshufb %xmm7, %xmm14, %xmm71444; AVX2-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,2,2]1445; AVX2-NEXT: vpshuflw {{.*#+}} xmm12 = xmm12[3,3,3,3,4,5,6,7]1446; AVX2-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,4,4,4]1447; AVX2-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1],xmm12[2],xmm9[3,4],xmm12[5],xmm9[6,7]1448; AVX2-NEXT: vinserti128 $1, %xmm7, %ymm9, %ymm71449; AVX2-NEXT: vpermd %ymm2, %ymm5, %ymm51450; AVX2-NEXT: vpblendvb %ymm8, %ymm7, %ymm5, %ymm51451; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[1,1,2,2]1452; AVX2-NEXT: vpshuflw {{.*#+}} xmm8 = xmm11[3,3,3,3,4,5,6,7]1453; AVX2-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,4,4,4]1454; AVX2-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm8[2],xmm7[3,4],xmm8[5],xmm7[6,7]1455; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm11[0],xmm6[1],xmm11[1],xmm6[2],xmm11[2],xmm6[3],xmm11[3]1456; AVX2-NEXT: vmovdqa {{.*#+}} xmm8 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]1457; AVX2-NEXT: vpshufb %xmm8, %xmm6, %xmm61458; AVX2-NEXT: vinserti128 $1, %xmm7, %ymm6, %ymm61459; AVX2-NEXT: vpmovsxbd {{.*#+}} ymm7 = [0,0,0,0,1,1,0,2]1460; AVX2-NEXT: vpermd %ymm2, %ymm7, %ymm91461; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm11 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]1462; AVX2-NEXT: vpblendvb %ymm11, %ymm6, %ymm9, %ymm61463; AVX2-NEXT: vpshufd {{.*#+}} xmm9 = xmm10[1,1,2,2]1464; AVX2-NEXT: vpshuflw {{.*#+}} xmm12 = xmm13[3,3,3,3,4,5,6,7]1465; AVX2-NEXT: vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,4,4,4]1466; AVX2-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1],xmm12[2],xmm9[3,4],xmm12[5],xmm9[6,7]1467; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm13[0],xmm10[1],xmm13[1],xmm10[2],xmm13[2],xmm10[3],xmm13[3]1468; AVX2-NEXT: vpshufb %xmm8, %xmm10, %xmm81469; AVX2-NEXT: vinserti128 $1, %xmm9, %ymm8, %ymm81470; AVX2-NEXT: vpermd %ymm1, %ymm7, %ymm71471; AVX2-NEXT: vpblendvb %ymm11, %ymm8, %ymm7, %ymm71472; AVX2-NEXT: vmovdqa {{.*#+}} ymm8 = [10,11,0,1,12,13,12,13,2,3,2,3,14,15,4,5,26,27,16,17,28,29,28,29,18,19,18,19,30,31,20,21]1473; AVX2-NEXT: vpshufb %ymm8, %ymm4, %ymm41474; AVX2-NEXT: vpmovsxbd {{.*#+}} ymm9 = [0,3,3,0,4,4,0,5]1475; AVX2-NEXT: vpermd (%rdi), %ymm9, %ymm101476; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm11 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]1477; AVX2-NEXT: vpblendvb %ymm11, %ymm4, %ymm10, %ymm41478; AVX2-NEXT: vpmovsxbd {{.*#+}} ymm10 = [2,0,3,3,0,4,4,0]1479; AVX2-NEXT: vpermd %ymm2, %ymm10, %ymm21480; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]1481; AVX2-NEXT: vpblendvb %ymm12, %ymm4, %ymm2, %ymm21482; AVX2-NEXT: vpshufb %ymm8, %ymm0, %ymm01483; AVX2-NEXT: vpermd 32(%rdi), %ymm9, %ymm41484; AVX2-NEXT: vpblendvb %ymm11, %ymm0, %ymm4, %ymm01485; AVX2-NEXT: vpermd %ymm1, %ymm10, %ymm11486; AVX2-NEXT: vpblendvb %ymm12, %ymm0, %ymm1, %ymm01487; AVX2-NEXT: vmovdqa %ymm0, 128(%rcx)1488; AVX2-NEXT: vmovdqa %ymm2, 32(%rcx)1489; AVX2-NEXT: vmovdqa %ymm7, 96(%rcx)1490; AVX2-NEXT: vmovdqa %ymm6, (%rcx)1491; AVX2-NEXT: vmovdqa %ymm5, 64(%rcx)1492; AVX2-NEXT: vmovdqa %ymm3, 160(%rcx)1493; AVX2-NEXT: vzeroupper1494; AVX2-NEXT: retq1495;1496; AVX2-FP-LABEL: store_i16_stride3_vf32:1497; AVX2-FP: # %bb.0:1498; AVX2-FP-NEXT: vmovdqa (%rdx), %ymm01499; AVX2-FP-NEXT: vmovdqa (%rsi), %xmm11500; AVX2-FP-NEXT: vmovdqa 32(%rsi), %xmm41501; AVX2-FP-NEXT: vpbroadcastd {{.*#+}} xmm2 = [6,7,8,9,6,7,8,9,6,7,8,9,6,7,8,9]1502; AVX2-FP-NEXT: vpshufb %xmm2, %xmm1, %xmm31503; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm51504; AVX2-FP-NEXT: vmovdqa 32(%rdi), %xmm61505; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,2,2]1506; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm7[0,1],xmm3[2],xmm7[3,4],xmm3[5],xmm7[6,7]1507; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]1508; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]1509; AVX2-FP-NEXT: vpshufb %xmm5, %xmm1, %xmm11510; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm11511; AVX2-FP-NEXT: vpmovsxbd {{.*#+}} ymm7 = [0,0,0,0,1,1,0,2]1512; AVX2-FP-NEXT: vpermd %ymm0, %ymm7, %ymm31513; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm8 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]1514; AVX2-FP-NEXT: vpblendvb %ymm8, %ymm1, %ymm3, %ymm11515; AVX2-FP-NEXT: vmovdqa 48(%rdi), %xmm31516; AVX2-FP-NEXT: vmovdqa 48(%rsi), %xmm91517; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm10 = xmm9[4],xmm3[4],xmm9[5],xmm3[5],xmm9[6],xmm3[6],xmm9[7],xmm3[7]1518; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm11 = [4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]1519; AVX2-FP-NEXT: vpshufb %xmm11, %xmm10, %xmm101520; AVX2-FP-NEXT: vpshufb %xmm2, %xmm9, %xmm91521; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,1,2,2]1522; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm9[2],xmm3[3,4],xmm9[5],xmm3[6,7]1523; AVX2-FP-NEXT: vinserti128 $1, %xmm10, %ymm3, %ymm91524; AVX2-FP-NEXT: vpshufb %xmm2, %xmm4, %xmm31525; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm10 = xmm6[1,1,2,2]1526; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1],xmm3[2],xmm10[3,4],xmm3[5],xmm10[6,7]1527; AVX2-FP-NEXT: vmovdqa 32(%rdx), %ymm31528; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]1529; AVX2-FP-NEXT: vmovdqa 16(%rsi), %xmm61530; AVX2-FP-NEXT: vpshufb %xmm5, %xmm4, %xmm41531; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm121532; AVX2-FP-NEXT: vinserti128 $1, %xmm10, %ymm4, %ymm41533; AVX2-FP-NEXT: vpmovsxbd {{.*#+}} ymm10 = [5,5,0,6,6,0,7,7]1534; AVX2-FP-NEXT: vpermd %ymm3, %ymm7, %ymm51535; AVX2-FP-NEXT: vpblendvb %ymm8, %ymm4, %ymm5, %ymm41536; AVX2-FP-NEXT: vpermd %ymm3, %ymm10, %ymm51537; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm7 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]1538; AVX2-FP-NEXT: vpblendvb %ymm7, %ymm9, %ymm5, %ymm51539; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm8 = xmm6[4],xmm12[4],xmm6[5],xmm12[5],xmm6[6],xmm12[6],xmm6[7],xmm12[7]1540; AVX2-FP-NEXT: vpshufb %xmm11, %xmm8, %xmm81541; AVX2-FP-NEXT: vpshufb %xmm2, %xmm6, %xmm21542; AVX2-FP-NEXT: vmovdqa (%rsi), %ymm61543; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm9 = xmm12[1,1,2,2]1544; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm2 = xmm9[0,1],xmm2[2],xmm9[3,4],xmm2[5],xmm9[6,7]1545; AVX2-FP-NEXT: vinserti128 $1, %xmm8, %ymm2, %ymm21546; AVX2-FP-NEXT: vpermd %ymm0, %ymm10, %ymm81547; AVX2-FP-NEXT: vpblendvb %ymm7, %ymm2, %ymm8, %ymm21548; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm7 = [10,11,0,1,12,13,12,13,2,3,2,3,14,15,4,5,26,27,16,17,28,29,28,29,18,19,18,19,30,31,20,21]1549; AVX2-FP-NEXT: vpshufb %ymm7, %ymm6, %ymm61550; AVX2-FP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [0,3,3,0,4,4,0,5]1551; AVX2-FP-NEXT: vpermd (%rdi), %ymm8, %ymm91552; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm10 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]1553; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm6, %ymm9, %ymm61554; AVX2-FP-NEXT: vmovdqa 32(%rsi), %ymm91555; AVX2-FP-NEXT: vpshufb %ymm7, %ymm9, %ymm71556; AVX2-FP-NEXT: vpermd 32(%rdi), %ymm8, %ymm81557; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm7, %ymm8, %ymm71558; AVX2-FP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [2,0,3,3,0,4,4,0]1559; AVX2-FP-NEXT: vpermd %ymm0, %ymm8, %ymm01560; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]1561; AVX2-FP-NEXT: vpblendvb %ymm9, %ymm6, %ymm0, %ymm01562; AVX2-FP-NEXT: vpermd %ymm3, %ymm8, %ymm31563; AVX2-FP-NEXT: vpblendvb %ymm9, %ymm7, %ymm3, %ymm31564; AVX2-FP-NEXT: vmovdqa %ymm3, 128(%rcx)1565; AVX2-FP-NEXT: vmovdqa %ymm0, 32(%rcx)1566; AVX2-FP-NEXT: vmovdqa %ymm2, 64(%rcx)1567; AVX2-FP-NEXT: vmovdqa %ymm4, 96(%rcx)1568; AVX2-FP-NEXT: vmovdqa %ymm5, 160(%rcx)1569; AVX2-FP-NEXT: vmovdqa %ymm1, (%rcx)1570; AVX2-FP-NEXT: vzeroupper1571; AVX2-FP-NEXT: retq1572;1573; AVX2-FCP-LABEL: store_i16_stride3_vf32:1574; AVX2-FCP: # %bb.0:1575; AVX2-FCP-NEXT: vmovdqa (%rdx), %ymm01576; AVX2-FCP-NEXT: vmovdqa (%rsi), %xmm11577; AVX2-FCP-NEXT: vmovdqa 32(%rsi), %xmm41578; AVX2-FCP-NEXT: vpbroadcastd {{.*#+}} xmm2 = [6,7,8,9,6,7,8,9,6,7,8,9,6,7,8,9]1579; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm1, %xmm31580; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm51581; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm61582; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,2,2]1583; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm7[0,1],xmm3[2],xmm7[3,4],xmm3[5],xmm7[6,7]1584; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]1585; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]1586; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm1, %xmm11587; AVX2-FCP-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm11588; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm7 = [0,0,0,0,1,1,0,2]1589; AVX2-FCP-NEXT: vpermd %ymm0, %ymm7, %ymm31590; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm8 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]1591; AVX2-FCP-NEXT: vpblendvb %ymm8, %ymm1, %ymm3, %ymm11592; AVX2-FCP-NEXT: vmovdqa 48(%rdi), %xmm31593; AVX2-FCP-NEXT: vmovdqa 48(%rsi), %xmm91594; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm10 = xmm9[4],xmm3[4],xmm9[5],xmm3[5],xmm9[6],xmm3[6],xmm9[7],xmm3[7]1595; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm11 = [4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]1596; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm10, %xmm101597; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm9, %xmm91598; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,1,2,2]1599; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm9[2],xmm3[3,4],xmm9[5],xmm3[6,7]1600; AVX2-FCP-NEXT: vinserti128 $1, %xmm10, %ymm3, %ymm91601; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm4, %xmm31602; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm10 = xmm6[1,1,2,2]1603; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1],xmm3[2],xmm10[3,4],xmm3[5],xmm10[6,7]1604; AVX2-FCP-NEXT: vmovdqa 32(%rdx), %ymm31605; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]1606; AVX2-FCP-NEXT: vmovdqa 16(%rsi), %xmm61607; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm4, %xmm41608; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm121609; AVX2-FCP-NEXT: vinserti128 $1, %xmm10, %ymm4, %ymm41610; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm10 = [5,5,0,6,6,0,7,7]1611; AVX2-FCP-NEXT: vpermd %ymm3, %ymm7, %ymm51612; AVX2-FCP-NEXT: vpblendvb %ymm8, %ymm4, %ymm5, %ymm41613; AVX2-FCP-NEXT: vpermd %ymm3, %ymm10, %ymm51614; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm7 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]1615; AVX2-FCP-NEXT: vpblendvb %ymm7, %ymm9, %ymm5, %ymm51616; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm8 = xmm6[4],xmm12[4],xmm6[5],xmm12[5],xmm6[6],xmm12[6],xmm6[7],xmm12[7]1617; AVX2-FCP-NEXT: vpshufb %xmm11, %xmm8, %xmm81618; AVX2-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm21619; AVX2-FCP-NEXT: vmovdqa (%rsi), %ymm61620; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm9 = xmm12[1,1,2,2]1621; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm2 = xmm9[0,1],xmm2[2],xmm9[3,4],xmm2[5],xmm9[6,7]1622; AVX2-FCP-NEXT: vinserti128 $1, %xmm8, %ymm2, %ymm21623; AVX2-FCP-NEXT: vpermd %ymm0, %ymm10, %ymm81624; AVX2-FCP-NEXT: vpblendvb %ymm7, %ymm2, %ymm8, %ymm21625; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [10,11,0,1,12,13,12,13,2,3,2,3,14,15,4,5,26,27,16,17,28,29,28,29,18,19,18,19,30,31,20,21]1626; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm6, %ymm61627; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [0,3,3,0,4,4,0,5]1628; AVX2-FCP-NEXT: vpermd (%rdi), %ymm8, %ymm91629; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm10 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]1630; AVX2-FCP-NEXT: vpblendvb %ymm10, %ymm6, %ymm9, %ymm61631; AVX2-FCP-NEXT: vmovdqa 32(%rsi), %ymm91632; AVX2-FCP-NEXT: vpshufb %ymm7, %ymm9, %ymm71633; AVX2-FCP-NEXT: vpermd 32(%rdi), %ymm8, %ymm81634; AVX2-FCP-NEXT: vpblendvb %ymm10, %ymm7, %ymm8, %ymm71635; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm8 = [2,0,3,3,0,4,4,0]1636; AVX2-FCP-NEXT: vpermd %ymm0, %ymm8, %ymm01637; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]1638; AVX2-FCP-NEXT: vpblendvb %ymm9, %ymm6, %ymm0, %ymm01639; AVX2-FCP-NEXT: vpermd %ymm3, %ymm8, %ymm31640; AVX2-FCP-NEXT: vpblendvb %ymm9, %ymm7, %ymm3, %ymm31641; AVX2-FCP-NEXT: vmovdqa %ymm3, 128(%rcx)1642; AVX2-FCP-NEXT: vmovdqa %ymm0, 32(%rcx)1643; AVX2-FCP-NEXT: vmovdqa %ymm2, 64(%rcx)1644; AVX2-FCP-NEXT: vmovdqa %ymm4, 96(%rcx)1645; AVX2-FCP-NEXT: vmovdqa %ymm5, 160(%rcx)1646; AVX2-FCP-NEXT: vmovdqa %ymm1, (%rcx)1647; AVX2-FCP-NEXT: vzeroupper1648; AVX2-FCP-NEXT: retq1649;1650; AVX512-LABEL: store_i16_stride3_vf32:1651; AVX512: # %bb.0:1652; AVX512-NEXT: vmovdqa (%rsi), %xmm01653; AVX512-NEXT: vmovdqa 16(%rsi), %xmm31654; AVX512-NEXT: vmovdqa 32(%rsi), %xmm21655; AVX512-NEXT: vprold $16, %xmm2, %xmm41656; AVX512-NEXT: vmovdqa (%rdi), %xmm11657; AVX512-NEXT: vmovdqa 16(%rdi), %xmm51658; AVX512-NEXT: vmovdqa 32(%rdi), %xmm61659; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[1,1,2,2]1660; AVX512-NEXT: vpblendw {{.*#+}} xmm4 = xmm7[0,1],xmm4[2],xmm7[3,4],xmm4[5],xmm7[6,7]1661; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]1662; AVX512-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]1663; AVX512-NEXT: vpshufb %xmm2, %xmm6, %xmm61664; AVX512-NEXT: vinserti128 $1, %xmm4, %ymm6, %ymm41665; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]1666; AVX512-NEXT: vmovdqa {{.*#+}} xmm7 = [4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]1667; AVX512-NEXT: vpshufb %xmm7, %xmm6, %xmm61668; AVX512-NEXT: vprold $16, %xmm3, %xmm31669; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,2]1670; AVX512-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2],xmm5[3,4],xmm3[5],xmm5[6,7]1671; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm3, %ymm31672; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm4[0,1,2,3]1673; AVX512-NEXT: vpmovsxbd {{.*#+}} zmm4 = [5,5,0,6,6,0,7,7,0,8,8,0,9,9,0,10]1674; AVX512-NEXT: vpermd (%rdx), %zmm4, %zmm51675; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm6 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]1676; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm6 & (zmm5 ^ zmm3))1677; AVX512-NEXT: vmovdqa 32(%rdi), %ymm31678; AVX512-NEXT: vmovdqa {{.*#+}} ymm8 = [128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128,20,21,128,128]1679; AVX512-NEXT: vpshufb %ymm8, %ymm3, %ymm31680; AVX512-NEXT: vmovdqa 32(%rsi), %ymm91681; AVX512-NEXT: vmovdqa {{.*#+}} ymm10 = [10,11,0,1,128,128,12,13,2,3,128,128,14,15,4,5,128,128,16,17,28,29,128,128,18,19,18,19,128,128,20,21]1682; AVX512-NEXT: vpshufb %ymm10, %ymm9, %ymm91683; AVX512-NEXT: vpor %ymm3, %ymm9, %ymm31684; AVX512-NEXT: vmovdqa 48(%rdi), %xmm91685; AVX512-NEXT: vmovdqa 48(%rsi), %xmm111686; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7]1687; AVX512-NEXT: vpshufb %xmm7, %xmm12, %xmm71688; AVX512-NEXT: vprold $16, %xmm11, %xmm111689; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,2,2]1690; AVX512-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1],xmm11[2],xmm9[3,4],xmm11[5],xmm9[6,7]1691; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm9, %ymm71692; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm7[0,1,2,3]1693; AVX512-NEXT: vmovdqa (%rdx), %ymm71694; AVX512-NEXT: vmovdqa 32(%rdx), %ymm91695; AVX512-NEXT: vpermd %ymm9, %ymm4, %ymm41696; AVX512-NEXT: vpandn %ymm4, %ymm6, %ymm41697; AVX512-NEXT: vmovdqa {{.*#+}} ymm6 = [128,128,10,11,128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128]1698; AVX512-NEXT: vpshufb %ymm6, %ymm9, %ymm91699; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm9, %zmm41700; AVX512-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm3 & mem)1701; AVX512-NEXT: vmovdqa (%rdi), %ymm31702; AVX512-NEXT: vpshufb %ymm8, %ymm3, %ymm31703; AVX512-NEXT: vmovdqa (%rsi), %ymm81704; AVX512-NEXT: vpshufb %ymm10, %ymm8, %ymm81705; AVX512-NEXT: vpor %ymm3, %ymm8, %ymm31706; AVX512-NEXT: vprold $16, %xmm0, %xmm81707; AVX512-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[1,1,2,2]1708; AVX512-NEXT: vpblendw {{.*#+}} xmm8 = xmm9[0,1],xmm8[2],xmm9[3,4],xmm8[5],xmm9[6,7]1709; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]1710; AVX512-NEXT: vpshufb %xmm2, %xmm0, %xmm01711; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm01712; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm3[0,1,2,3]1713; AVX512-NEXT: vpshufb %ymm6, %ymm7, %ymm11714; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,0,0,0,1,1,0,2]1715; AVX512-NEXT: vpermd %ymm7, %ymm2, %ymm21716; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm21717; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm11718; AVX512-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & mem)1719; AVX512-NEXT: vmovdqa64 %zmm1, (%rcx)1720; AVX512-NEXT: vmovdqa64 %zmm4, 128(%rcx)1721; AVX512-NEXT: vmovdqa64 %zmm5, 64(%rcx)1722; AVX512-NEXT: vzeroupper1723; AVX512-NEXT: retq1724;1725; AVX512-FCP-LABEL: store_i16_stride3_vf32:1726; AVX512-FCP: # %bb.0:1727; AVX512-FCP-NEXT: vmovdqa (%rsi), %xmm01728; AVX512-FCP-NEXT: vmovdqa 16(%rsi), %xmm31729; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %xmm21730; AVX512-FCP-NEXT: vprold $16, %xmm2, %xmm41731; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm11732; AVX512-FCP-NEXT: vmovdqa 16(%rdi), %xmm51733; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %xmm61734; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[1,1,2,2]1735; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm7[0,1],xmm4[2],xmm7[3,4],xmm4[5],xmm7[6,7]1736; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]1737; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]1738; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm61739; AVX512-FCP-NEXT: vinserti128 $1, %xmm4, %ymm6, %ymm41740; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]1741; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm7 = [4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]1742; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm6, %xmm61743; AVX512-FCP-NEXT: vprold $16, %xmm3, %xmm31744; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,2]1745; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2],xmm5[3,4],xmm3[5],xmm5[6,7]1746; AVX512-FCP-NEXT: vinserti128 $1, %xmm6, %ymm3, %ymm31747; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm4[0,1,2,3]1748; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm4 = [5,5,0,6,6,0,7,7,0,8,8,0,9,9,0,10]1749; AVX512-FCP-NEXT: vpermd (%rdx), %zmm4, %zmm51750; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm6 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]1751; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm6 & (zmm5 ^ zmm3))1752; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm31753; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm8 = [128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128,20,21,128,128]1754; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm3, %ymm31755; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %ymm91756; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm10 = [10,11,0,1,128,128,12,13,2,3,128,128,14,15,4,5,128,128,16,17,28,29,128,128,18,19,18,19,128,128,20,21]1757; AVX512-FCP-NEXT: vpshufb %ymm10, %ymm9, %ymm91758; AVX512-FCP-NEXT: vpor %ymm3, %ymm9, %ymm31759; AVX512-FCP-NEXT: vmovdqa 48(%rdi), %xmm91760; AVX512-FCP-NEXT: vmovdqa 48(%rsi), %xmm111761; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7]1762; AVX512-FCP-NEXT: vpshufb %xmm7, %xmm12, %xmm71763; AVX512-FCP-NEXT: vprold $16, %xmm11, %xmm111764; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,2,2]1765; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1],xmm11[2],xmm9[3,4],xmm11[5],xmm9[6,7]1766; AVX512-FCP-NEXT: vinserti128 $1, %xmm7, %ymm9, %ymm71767; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm7[0,1,2,3]1768; AVX512-FCP-NEXT: vmovdqa (%rdx), %ymm71769; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %ymm91770; AVX512-FCP-NEXT: vpermd %ymm9, %ymm4, %ymm41771; AVX512-FCP-NEXT: vpandn %ymm4, %ymm6, %ymm41772; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [128,128,10,11,128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128]1773; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm9, %ymm91774; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm4, %zmm9, %zmm41775; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm3 & mem)1776; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm31777; AVX512-FCP-NEXT: vpshufb %ymm8, %ymm3, %ymm31778; AVX512-FCP-NEXT: vmovdqa (%rsi), %ymm81779; AVX512-FCP-NEXT: vpshufb %ymm10, %ymm8, %ymm81780; AVX512-FCP-NEXT: vpor %ymm3, %ymm8, %ymm31781; AVX512-FCP-NEXT: vprold $16, %xmm0, %xmm81782; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[1,1,2,2]1783; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm8 = xmm9[0,1],xmm8[2],xmm9[3,4],xmm8[5],xmm9[6,7]1784; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]1785; AVX512-FCP-NEXT: vpshufb %xmm2, %xmm0, %xmm01786; AVX512-FCP-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm01787; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm3[0,1,2,3]1788; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm7, %ymm11789; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,0,0,0,1,1,0,2]1790; AVX512-FCP-NEXT: vpermd %ymm7, %ymm2, %ymm21791; AVX512-FCP-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm21792; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm11793; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & mem)1794; AVX512-FCP-NEXT: vmovdqa64 %zmm1, (%rcx)1795; AVX512-FCP-NEXT: vmovdqa64 %zmm4, 128(%rcx)1796; AVX512-FCP-NEXT: vmovdqa64 %zmm5, 64(%rcx)1797; AVX512-FCP-NEXT: vzeroupper1798; AVX512-FCP-NEXT: retq1799;1800; AVX512DQ-LABEL: store_i16_stride3_vf32:1801; AVX512DQ: # %bb.0:1802; AVX512DQ-NEXT: vmovdqa (%rsi), %xmm01803; AVX512DQ-NEXT: vmovdqa 16(%rsi), %xmm31804; AVX512DQ-NEXT: vmovdqa 32(%rsi), %xmm21805; AVX512DQ-NEXT: vprold $16, %xmm2, %xmm41806; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm11807; AVX512DQ-NEXT: vmovdqa 16(%rdi), %xmm51808; AVX512DQ-NEXT: vmovdqa 32(%rdi), %xmm61809; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[1,1,2,2]1810; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm4 = xmm7[0,1],xmm4[2],xmm7[3,4],xmm4[5],xmm7[6,7]1811; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]1812; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]1813; AVX512DQ-NEXT: vpshufb %xmm2, %xmm6, %xmm61814; AVX512DQ-NEXT: vinserti128 $1, %xmm4, %ymm6, %ymm41815; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]1816; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm7 = [4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]1817; AVX512DQ-NEXT: vpshufb %xmm7, %xmm6, %xmm61818; AVX512DQ-NEXT: vprold $16, %xmm3, %xmm31819; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,2]1820; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2],xmm5[3,4],xmm3[5],xmm5[6,7]1821; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm3, %ymm31822; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm4[0,1,2,3]1823; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm4 = [5,5,0,6,6,0,7,7,0,8,8,0,9,9,0,10]1824; AVX512DQ-NEXT: vpermd (%rdx), %zmm4, %zmm51825; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm6 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]1826; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm6 & (zmm5 ^ zmm3))1827; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm31828; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm8 = [128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128,20,21,128,128]1829; AVX512DQ-NEXT: vpshufb %ymm8, %ymm3, %ymm31830; AVX512DQ-NEXT: vmovdqa 32(%rsi), %ymm91831; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm10 = [10,11,0,1,128,128,12,13,2,3,128,128,14,15,4,5,128,128,16,17,28,29,128,128,18,19,18,19,128,128,20,21]1832; AVX512DQ-NEXT: vpshufb %ymm10, %ymm9, %ymm91833; AVX512DQ-NEXT: vpor %ymm3, %ymm9, %ymm31834; AVX512DQ-NEXT: vmovdqa 48(%rdi), %xmm91835; AVX512DQ-NEXT: vmovdqa 48(%rsi), %xmm111836; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7]1837; AVX512DQ-NEXT: vpshufb %xmm7, %xmm12, %xmm71838; AVX512DQ-NEXT: vprold $16, %xmm11, %xmm111839; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,2,2]1840; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1],xmm11[2],xmm9[3,4],xmm11[5],xmm9[6,7]1841; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm9, %ymm71842; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm7[0,1,2,3]1843; AVX512DQ-NEXT: vmovdqa (%rdx), %ymm71844; AVX512DQ-NEXT: vmovdqa 32(%rdx), %ymm91845; AVX512DQ-NEXT: vpermd %ymm9, %ymm4, %ymm41846; AVX512DQ-NEXT: vpandn %ymm4, %ymm6, %ymm41847; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm6 = [128,128,10,11,128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128]1848; AVX512DQ-NEXT: vpshufb %ymm6, %ymm9, %ymm91849; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm9, %zmm41850; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm3 & mem)1851; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm31852; AVX512DQ-NEXT: vpshufb %ymm8, %ymm3, %ymm31853; AVX512DQ-NEXT: vmovdqa (%rsi), %ymm81854; AVX512DQ-NEXT: vpshufb %ymm10, %ymm8, %ymm81855; AVX512DQ-NEXT: vpor %ymm3, %ymm8, %ymm31856; AVX512DQ-NEXT: vprold $16, %xmm0, %xmm81857; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[1,1,2,2]1858; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm8 = xmm9[0,1],xmm8[2],xmm9[3,4],xmm8[5],xmm9[6,7]1859; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]1860; AVX512DQ-NEXT: vpshufb %xmm2, %xmm0, %xmm01861; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm01862; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm3[0,1,2,3]1863; AVX512DQ-NEXT: vpshufb %ymm6, %ymm7, %ymm11864; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,0,0,0,1,1,0,2]1865; AVX512DQ-NEXT: vpermd %ymm7, %ymm2, %ymm21866; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm21867; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm11868; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & mem)1869; AVX512DQ-NEXT: vmovdqa64 %zmm1, (%rcx)1870; AVX512DQ-NEXT: vmovdqa64 %zmm4, 128(%rcx)1871; AVX512DQ-NEXT: vmovdqa64 %zmm5, 64(%rcx)1872; AVX512DQ-NEXT: vzeroupper1873; AVX512DQ-NEXT: retq1874;1875; AVX512DQ-FCP-LABEL: store_i16_stride3_vf32:1876; AVX512DQ-FCP: # %bb.0:1877; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %xmm01878; AVX512DQ-FCP-NEXT: vmovdqa 16(%rsi), %xmm31879; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %xmm21880; AVX512DQ-FCP-NEXT: vprold $16, %xmm2, %xmm41881; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm11882; AVX512DQ-FCP-NEXT: vmovdqa 16(%rdi), %xmm51883; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %xmm61884; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[1,1,2,2]1885; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm7[0,1],xmm4[2],xmm7[3,4],xmm4[5],xmm7[6,7]1886; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]1887; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]1888; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm6, %xmm61889; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm4, %ymm6, %ymm41890; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]1891; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm7 = [4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]1892; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm6, %xmm61893; AVX512DQ-FCP-NEXT: vprold $16, %xmm3, %xmm31894; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,2]1895; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm5[0,1],xmm3[2],xmm5[3,4],xmm3[5],xmm5[6,7]1896; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm6, %ymm3, %ymm31897; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm4[0,1,2,3]1898; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm4 = [5,5,0,6,6,0,7,7,0,8,8,0,9,9,0,10]1899; AVX512DQ-FCP-NEXT: vpermd (%rdx), %zmm4, %zmm51900; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm6 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]1901; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 ^ (zmm6 & (zmm5 ^ zmm3))1902; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm31903; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm8 = [128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128,20,21,128,128]1904; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm3, %ymm31905; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %ymm91906; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm10 = [10,11,0,1,128,128,12,13,2,3,128,128,14,15,4,5,128,128,16,17,28,29,128,128,18,19,18,19,128,128,20,21]1907; AVX512DQ-FCP-NEXT: vpshufb %ymm10, %ymm9, %ymm91908; AVX512DQ-FCP-NEXT: vpor %ymm3, %ymm9, %ymm31909; AVX512DQ-FCP-NEXT: vmovdqa 48(%rdi), %xmm91910; AVX512DQ-FCP-NEXT: vmovdqa 48(%rsi), %xmm111911; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7]1912; AVX512DQ-FCP-NEXT: vpshufb %xmm7, %xmm12, %xmm71913; AVX512DQ-FCP-NEXT: vprold $16, %xmm11, %xmm111914; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,2,2]1915; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1],xmm11[2],xmm9[3,4],xmm11[5],xmm9[6,7]1916; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm7, %ymm9, %ymm71917; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm7[0,1,2,3]1918; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %ymm71919; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %ymm91920; AVX512DQ-FCP-NEXT: vpermd %ymm9, %ymm4, %ymm41921; AVX512DQ-FCP-NEXT: vpandn %ymm4, %ymm6, %ymm41922; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [128,128,10,11,128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128]1923; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm9, %ymm91924; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm4, %zmm9, %zmm41925; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm4 = zmm4 | (zmm3 & mem)1926; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm31927; AVX512DQ-FCP-NEXT: vpshufb %ymm8, %ymm3, %ymm31928; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %ymm81929; AVX512DQ-FCP-NEXT: vpshufb %ymm10, %ymm8, %ymm81930; AVX512DQ-FCP-NEXT: vpor %ymm3, %ymm8, %ymm31931; AVX512DQ-FCP-NEXT: vprold $16, %xmm0, %xmm81932; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[1,1,2,2]1933; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm8 = xmm9[0,1],xmm8[2],xmm9[3,4],xmm8[5],xmm9[6,7]1934; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]1935; AVX512DQ-FCP-NEXT: vpshufb %xmm2, %xmm0, %xmm01936; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm8, %ymm0, %ymm01937; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm3[0,1,2,3]1938; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm7, %ymm11939; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm2 = [0,0,0,0,1,1,0,2]1940; AVX512DQ-FCP-NEXT: vpermd %ymm7, %ymm2, %ymm21941; AVX512DQ-FCP-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm21942; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm11943; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 | (zmm0 & mem)1944; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, (%rcx)1945; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm4, 128(%rcx)1946; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, 64(%rcx)1947; AVX512DQ-FCP-NEXT: vzeroupper1948; AVX512DQ-FCP-NEXT: retq1949;1950; AVX512BW-LABEL: store_i16_stride3_vf32:1951; AVX512BW: # %bb.0:1952; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm01953; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm11954; AVX512BW-NEXT: vmovdqa64 (%rdx), %zmm21955; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,32,0,1,33,0,2,34,0,3,35,0,4,36,0,5,37,0,6,38,0,7,39,0,8,40,0,9,41,0,10,42]1956; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm31957; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,1,32,3,4,33,6,7,34,9,10,35,12,13,36,15,16,37,18,19,38,21,22,39,24,25,40,27,28,41,30,31]1958; AVX512BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm41959; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,11,43,0,12,44,0,13,45,0,14,46,0,15,47,0,16,48,0,17,49,0,18,50,0,19,51,0,20,52,0,21]1960; AVX512BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm31961; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm5 = [42,1,2,43,4,5,44,7,8,45,10,11,46,13,14,47,16,17,48,19,20,49,22,23,50,25,26,51,28,29,52,31]1962; AVX512BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm51963; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [21,0,54,22,0,55,23,0,56,24,0,57,25,0,58,26,0,59,27,0,60,28,0,61,29,0,62,30,0,63,31,0]1964; AVX512BW-NEXT: vpermi2w %zmm0, %zmm1, %zmm31965; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm0 = [0,53,2,3,54,5,6,55,8,9,56,11,12,57,14,15,58,17,18,59,20,21,60,23,24,61,26,27,62,29,30,63]1966; AVX512BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm01967; AVX512BW-NEXT: vmovdqa64 %zmm0, 128(%rcx)1968; AVX512BW-NEXT: vmovdqa64 %zmm5, 64(%rcx)1969; AVX512BW-NEXT: vmovdqa64 %zmm4, (%rcx)1970; AVX512BW-NEXT: vzeroupper1971; AVX512BW-NEXT: retq1972;1973; AVX512BW-FCP-LABEL: store_i16_stride3_vf32:1974; AVX512BW-FCP: # %bb.0:1975; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm01976; AVX512BW-FCP-NEXT: vmovdqa64 (%rsi), %zmm11977; AVX512BW-FCP-NEXT: vmovdqa64 (%rdx), %zmm21978; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,32,0,1,33,0,2,34,0,3,35,0,4,36,0,5,37,0,6,38,0,7,39,0,8,40,0,9,41,0,10,42]1979; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm31980; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,1,32,3,4,33,6,7,34,9,10,35,12,13,36,15,16,37,18,19,38,21,22,39,24,25,40,27,28,41,30,31]1981; AVX512BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm41982; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,11,43,0,12,44,0,13,45,0,14,46,0,15,47,0,16,48,0,17,49,0,18,50,0,19,51,0,20,52,0,21]1983; AVX512BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm31984; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm5 = [42,1,2,43,4,5,44,7,8,45,10,11,46,13,14,47,16,17,48,19,20,49,22,23,50,25,26,51,28,29,52,31]1985; AVX512BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm51986; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [21,0,54,22,0,55,23,0,56,24,0,57,25,0,58,26,0,59,27,0,60,28,0,61,29,0,62,30,0,63,31,0]1987; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm1, %zmm31988; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm0 = [0,53,2,3,54,5,6,55,8,9,56,11,12,57,14,15,58,17,18,59,20,21,60,23,24,61,26,27,62,29,30,63]1989; AVX512BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm01990; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, 128(%rcx)1991; AVX512BW-FCP-NEXT: vmovdqa64 %zmm5, 64(%rcx)1992; AVX512BW-FCP-NEXT: vmovdqa64 %zmm4, (%rcx)1993; AVX512BW-FCP-NEXT: vzeroupper1994; AVX512BW-FCP-NEXT: retq1995;1996; AVX512DQ-BW-LABEL: store_i16_stride3_vf32:1997; AVX512DQ-BW: # %bb.0:1998; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm01999; AVX512DQ-BW-NEXT: vmovdqa64 (%rsi), %zmm12000; AVX512DQ-BW-NEXT: vmovdqa64 (%rdx), %zmm22001; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,32,0,1,33,0,2,34,0,3,35,0,4,36,0,5,37,0,6,38,0,7,39,0,8,40,0,9,41,0,10,42]2002; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm32003; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,1,32,3,4,33,6,7,34,9,10,35,12,13,36,15,16,37,18,19,38,21,22,39,24,25,40,27,28,41,30,31]2004; AVX512DQ-BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm42005; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,11,43,0,12,44,0,13,45,0,14,46,0,15,47,0,16,48,0,17,49,0,18,50,0,19,51,0,20,52,0,21]2006; AVX512DQ-BW-NEXT: vpermi2w %zmm1, %zmm0, %zmm32007; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm5 = [42,1,2,43,4,5,44,7,8,45,10,11,46,13,14,47,16,17,48,19,20,49,22,23,50,25,26,51,28,29,52,31]2008; AVX512DQ-BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm52009; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm3 = [21,0,54,22,0,55,23,0,56,24,0,57,25,0,58,26,0,59,27,0,60,28,0,61,29,0,62,30,0,63,31,0]2010; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm1, %zmm32011; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm0 = [0,53,2,3,54,5,6,55,8,9,56,11,12,57,14,15,58,17,18,59,20,21,60,23,24,61,26,27,62,29,30,63]2012; AVX512DQ-BW-NEXT: vpermi2w %zmm2, %zmm3, %zmm02013; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, 128(%rcx)2014; AVX512DQ-BW-NEXT: vmovdqa64 %zmm5, 64(%rcx)2015; AVX512DQ-BW-NEXT: vmovdqa64 %zmm4, (%rcx)2016; AVX512DQ-BW-NEXT: vzeroupper2017; AVX512DQ-BW-NEXT: retq2018;2019; AVX512DQ-BW-FCP-LABEL: store_i16_stride3_vf32:2020; AVX512DQ-BW-FCP: # %bb.0:2021; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm02022; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rsi), %zmm12023; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdx), %zmm22024; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,32,0,1,33,0,2,34,0,3,35,0,4,36,0,5,37,0,6,38,0,7,39,0,8,40,0,9,41,0,10,42]2025; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm32026; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm4 = [0,1,32,3,4,33,6,7,34,9,10,35,12,13,36,15,16,37,18,19,38,21,22,39,24,25,40,27,28,41,30,31]2027; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm42028; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [0,11,43,0,12,44,0,13,45,0,14,46,0,15,47,0,16,48,0,17,49,0,18,50,0,19,51,0,20,52,0,21]2029; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm1, %zmm0, %zmm32030; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm5 = [42,1,2,43,4,5,44,7,8,45,10,11,46,13,14,47,16,17,48,19,20,49,22,23,50,25,26,51,28,29,52,31]2031; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm52032; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm3 = [21,0,54,22,0,55,23,0,56,24,0,57,25,0,58,26,0,59,27,0,60,28,0,61,29,0,62,30,0,63,31,0]2033; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm1, %zmm32034; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm0 = [0,53,2,3,54,5,6,55,8,9,56,11,12,57,14,15,58,17,18,59,20,21,60,23,24,61,26,27,62,29,30,63]2035; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm2, %zmm3, %zmm02036; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, 128(%rcx)2037; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm5, 64(%rcx)2038; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm4, (%rcx)2039; AVX512DQ-BW-FCP-NEXT: vzeroupper2040; AVX512DQ-BW-FCP-NEXT: retq2041 %in.vec0 = load <32 x i16>, ptr %in.vecptr0, align 642042 %in.vec1 = load <32 x i16>, ptr %in.vecptr1, align 642043 %in.vec2 = load <32 x i16>, ptr %in.vecptr2, align 642044 %1 = shufflevector <32 x i16> %in.vec0, <32 x i16> %in.vec1, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>2045 %2 = shufflevector <32 x i16> %in.vec2, <32 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>2046 %3 = shufflevector <64 x i16> %1, <64 x i16> %2, <96 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>2047 %interleaved.vec = shufflevector <96 x i16> %3, <96 x i16> poison, <96 x i32> <i32 0, i32 32, i32 64, i32 1, i32 33, i32 65, i32 2, i32 34, i32 66, i32 3, i32 35, i32 67, i32 4, i32 36, i32 68, i32 5, i32 37, i32 69, i32 6, i32 38, i32 70, i32 7, i32 39, i32 71, i32 8, i32 40, i32 72, i32 9, i32 41, i32 73, i32 10, i32 42, i32 74, i32 11, i32 43, i32 75, i32 12, i32 44, i32 76, i32 13, i32 45, i32 77, i32 14, i32 46, i32 78, i32 15, i32 47, i32 79, i32 16, i32 48, i32 80, i32 17, i32 49, i32 81, i32 18, i32 50, i32 82, i32 19, i32 51, i32 83, i32 20, i32 52, i32 84, i32 21, i32 53, i32 85, i32 22, i32 54, i32 86, i32 23, i32 55, i32 87, i32 24, i32 56, i32 88, i32 25, i32 57, i32 89, i32 26, i32 58, i32 90, i32 27, i32 59, i32 91, i32 28, i32 60, i32 92, i32 29, i32 61, i32 93, i32 30, i32 62, i32 94, i32 31, i32 63, i32 95>2048 store <96 x i16> %interleaved.vec, ptr %out.vec, align 642049 ret void2050}2051 2052define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %out.vec) nounwind {2053; SSE-LABEL: store_i16_stride3_vf64:2054; SSE: # %bb.0:2055; SSE-NEXT: subq $328, %rsp # imm = 0x1482056; SSE-NEXT: movdqa (%rdi), %xmm32057; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2058; SSE-NEXT: movdqa 16(%rdi), %xmm82059; SSE-NEXT: movdqa 32(%rdi), %xmm62060; SSE-NEXT: movdqa (%rsi), %xmm42061; SSE-NEXT: movdqa 16(%rsi), %xmm52062; SSE-NEXT: movdqa (%rdx), %xmm102063; SSE-NEXT: movdqa 16(%rdx), %xmm92064; SSE-NEXT: movdqa 32(%rdx), %xmm72065; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2066; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm10[0,0,0,0]2067; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2068; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,0,65535,65535,0,65535,65535]2069; SSE-NEXT: movdqa %xmm0, %xmm22070; SSE-NEXT: pandn %xmm1, %xmm22071; SSE-NEXT: movdqa %xmm3, %xmm12072; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]2073; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]2074; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,2,4,5,6,7]2075; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,4,5]2076; SSE-NEXT: pand %xmm0, %xmm12077; SSE-NEXT: por %xmm2, %xmm12078; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2079; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,2,2]2080; SSE-NEXT: pand %xmm0, %xmm12081; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm4[3,3,3,3,4,5,6,7]2082; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,4,4]2083; SSE-NEXT: movdqa %xmm0, %xmm32084; SSE-NEXT: pandn %xmm2, %xmm32085; SSE-NEXT: por %xmm1, %xmm32086; SSE-NEXT: movdqa {{.*#+}} xmm11 = [0,65535,65535,0,65535,65535,0,65535]2087; SSE-NEXT: pand %xmm11, %xmm32088; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm10[1,1,2,2]2089; SSE-NEXT: movdqa %xmm11, %xmm22090; SSE-NEXT: pandn %xmm1, %xmm22091; SSE-NEXT: por %xmm3, %xmm22092; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2093; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm9[0,0,0,0]2094; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2095; SSE-NEXT: movdqa %xmm0, %xmm22096; SSE-NEXT: pandn %xmm1, %xmm22097; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2098; SSE-NEXT: movdqa %xmm8, %xmm12099; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]2100; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]2101; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,2,4,5,6,7]2102; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,4,5]2103; SSE-NEXT: pand %xmm0, %xmm12104; SSE-NEXT: por %xmm2, %xmm12105; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2106; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm8[1,1,2,2]2107; SSE-NEXT: pand %xmm0, %xmm12108; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm5[3,3,3,3,4,5,6,7]2109; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,4,4]2110; SSE-NEXT: movdqa %xmm0, %xmm32111; SSE-NEXT: pandn %xmm2, %xmm32112; SSE-NEXT: por %xmm1, %xmm32113; SSE-NEXT: pand %xmm11, %xmm32114; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm9[1,1,2,2]2115; SSE-NEXT: movdqa %xmm11, %xmm22116; SSE-NEXT: pandn %xmm1, %xmm22117; SSE-NEXT: por %xmm3, %xmm22118; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2119; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[0,0,0,0]2120; SSE-NEXT: movdqa %xmm0, %xmm22121; SSE-NEXT: pandn %xmm1, %xmm22122; SSE-NEXT: movdqa 32(%rsi), %xmm82123; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2124; SSE-NEXT: movdqa %xmm6, %xmm12125; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3]2126; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2127; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]2128; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,2,4,5,6,7]2129; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,4,5]2130; SSE-NEXT: pand %xmm0, %xmm12131; SSE-NEXT: por %xmm2, %xmm12132; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2133; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm8[3,3,3,3,4,5,6,7]2134; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]2135; SSE-NEXT: movdqa %xmm0, %xmm22136; SSE-NEXT: pandn %xmm1, %xmm22137; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[1,1,2,2]2138; SSE-NEXT: pand %xmm0, %xmm12139; SSE-NEXT: por %xmm1, %xmm22140; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[1,1,2,2]2141; SSE-NEXT: movdqa %xmm11, %xmm62142; SSE-NEXT: pandn %xmm1, %xmm62143; SSE-NEXT: pand %xmm11, %xmm22144; SSE-NEXT: por %xmm2, %xmm62145; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2146; SSE-NEXT: movdqa 48(%rdx), %xmm72147; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[0,0,0,0]2148; SSE-NEXT: movdqa %xmm7, (%rsp) # 16-byte Spill2149; SSE-NEXT: movdqa %xmm0, %xmm22150; SSE-NEXT: pandn %xmm1, %xmm22151; SSE-NEXT: movdqa 48(%rdi), %xmm62152; SSE-NEXT: movdqa 48(%rsi), %xmm32153; SSE-NEXT: movdqa %xmm6, %xmm12154; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2155; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]2156; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]2157; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,2,4,5,6,7]2158; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,4,5]2159; SSE-NEXT: pand %xmm0, %xmm12160; SSE-NEXT: por %xmm2, %xmm12161; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2162; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm3[3,3,3,3,4,5,6,7]2163; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]2164; SSE-NEXT: movdqa %xmm0, %xmm22165; SSE-NEXT: pandn %xmm1, %xmm22166; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[1,1,2,2]2167; SSE-NEXT: pand %xmm0, %xmm12168; SSE-NEXT: por %xmm1, %xmm22169; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[1,1,2,2]2170; SSE-NEXT: movdqa %xmm11, %xmm62171; SSE-NEXT: pandn %xmm1, %xmm62172; SSE-NEXT: pand %xmm11, %xmm22173; SSE-NEXT: por %xmm2, %xmm62174; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2175; SSE-NEXT: movdqa 64(%rdx), %xmm72176; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[0,0,0,0]2177; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2178; SSE-NEXT: movdqa %xmm0, %xmm22179; SSE-NEXT: pandn %xmm1, %xmm22180; SSE-NEXT: movdqa 64(%rdi), %xmm62181; SSE-NEXT: movdqa 64(%rsi), %xmm142182; SSE-NEXT: movdqa %xmm6, %xmm12183; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2184; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm14[0],xmm1[1],xmm14[1],xmm1[2],xmm14[2],xmm1[3],xmm14[3]2185; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]2186; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,2,4,5,6,7]2187; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,4,5]2188; SSE-NEXT: pand %xmm0, %xmm12189; SSE-NEXT: por %xmm2, %xmm12190; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2191; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm14[3,3,3,3,4,5,6,7]2192; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]2193; SSE-NEXT: movdqa %xmm0, %xmm22194; SSE-NEXT: pandn %xmm1, %xmm22195; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[1,1,2,2]2196; SSE-NEXT: pand %xmm0, %xmm12197; SSE-NEXT: por %xmm1, %xmm22198; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[1,1,2,2]2199; SSE-NEXT: movdqa %xmm11, %xmm62200; SSE-NEXT: pandn %xmm1, %xmm62201; SSE-NEXT: pand %xmm11, %xmm22202; SSE-NEXT: por %xmm2, %xmm62203; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2204; SSE-NEXT: movdqa 80(%rdx), %xmm72205; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[0,0,0,0]2206; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2207; SSE-NEXT: movdqa %xmm0, %xmm22208; SSE-NEXT: pandn %xmm1, %xmm22209; SSE-NEXT: movdqa 80(%rdi), %xmm62210; SSE-NEXT: movdqa 80(%rsi), %xmm122211; SSE-NEXT: movdqa %xmm6, %xmm12212; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2213; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm12[0],xmm1[1],xmm12[1],xmm1[2],xmm12[2],xmm1[3],xmm12[3]2214; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]2215; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,2,4,5,6,7]2216; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,4,5]2217; SSE-NEXT: pand %xmm0, %xmm12218; SSE-NEXT: por %xmm2, %xmm12219; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2220; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm12[3,3,3,3,4,5,6,7]2221; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]2222; SSE-NEXT: movdqa %xmm0, %xmm22223; SSE-NEXT: pandn %xmm1, %xmm22224; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[1,1,2,2]2225; SSE-NEXT: pand %xmm0, %xmm12226; SSE-NEXT: por %xmm1, %xmm22227; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[1,1,2,2]2228; SSE-NEXT: movdqa %xmm11, %xmm62229; SSE-NEXT: pandn %xmm1, %xmm62230; SSE-NEXT: pand %xmm11, %xmm22231; SSE-NEXT: por %xmm2, %xmm62232; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2233; SSE-NEXT: movdqa 96(%rdx), %xmm72234; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[0,0,0,0]2235; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2236; SSE-NEXT: movdqa %xmm0, %xmm22237; SSE-NEXT: pandn %xmm1, %xmm22238; SSE-NEXT: movdqa 96(%rdi), %xmm62239; SSE-NEXT: movdqa 96(%rsi), %xmm92240; SSE-NEXT: movdqa %xmm6, %xmm12241; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2242; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm9[0],xmm1[1],xmm9[1],xmm1[2],xmm9[2],xmm1[3],xmm9[3]2243; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]2244; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,2,4,5,6,7]2245; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm1[0,1,2,3,7,5,4,5]2246; SSE-NEXT: pand %xmm0, %xmm152247; SSE-NEXT: por %xmm2, %xmm152248; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm9[3,3,3,3,4,5,6,7]2249; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]2250; SSE-NEXT: movdqa %xmm0, %xmm22251; SSE-NEXT: pandn %xmm1, %xmm22252; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[1,1,2,2]2253; SSE-NEXT: pand %xmm0, %xmm12254; SSE-NEXT: por %xmm1, %xmm22255; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm7[1,1,2,2]2256; SSE-NEXT: movdqa %xmm11, %xmm132257; SSE-NEXT: pandn %xmm1, %xmm132258; SSE-NEXT: pand %xmm11, %xmm22259; SSE-NEXT: por %xmm2, %xmm132260; SSE-NEXT: movdqa 112(%rdx), %xmm62261; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[0,0,0,0]2262; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2263; SSE-NEXT: movdqa %xmm0, %xmm22264; SSE-NEXT: pandn %xmm1, %xmm22265; SSE-NEXT: movdqa 112(%rdi), %xmm82266; SSE-NEXT: movdqa 112(%rsi), %xmm72267; SSE-NEXT: movdqa %xmm8, %xmm12268; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm7[0],xmm1[1],xmm7[1],xmm1[2],xmm7[2],xmm1[3],xmm7[3]2269; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1]2270; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,2,4,5,6,7]2271; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm1[0,1,2,3,7,5,4,5]2272; SSE-NEXT: pand %xmm0, %xmm102273; SSE-NEXT: por %xmm2, %xmm102274; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm8[1,1,2,2]2275; SSE-NEXT: pand %xmm0, %xmm12276; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm7[3,3,3,3,4,5,6,7]2277; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,4,4]2278; SSE-NEXT: pandn %xmm2, %xmm02279; SSE-NEXT: por %xmm1, %xmm02280; SSE-NEXT: pand %xmm11, %xmm02281; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[1,1,2,2]2282; SSE-NEXT: pandn %xmm1, %xmm112283; SSE-NEXT: por %xmm0, %xmm112284; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload2285; SSE-NEXT: # xmm1 = mem[2,2,3,3]2286; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,0,65535,65535,0,65535,65535,0]2287; SSE-NEXT: movdqa %xmm0, %xmm22288; SSE-NEXT: pandn %xmm1, %xmm22289; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload2290; SSE-NEXT: # xmm4 = xmm4[4],mem[4],xmm4[5],mem[5],xmm4[6],mem[6],xmm4[7],mem[7]2291; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm4[2,1,3,3]2292; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm1[2,1,1,0,4,5,6,7]2293; SSE-NEXT: pand %xmm0, %xmm62294; SSE-NEXT: por %xmm2, %xmm62295; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload2296; SSE-NEXT: # xmm1 = mem[2,2,3,3]2297; SSE-NEXT: movdqa %xmm0, %xmm22298; SSE-NEXT: pandn %xmm1, %xmm22299; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload2300; SSE-NEXT: # xmm5 = xmm5[4],mem[4],xmm5[5],mem[5],xmm5[6],mem[6],xmm5[7],mem[7]2301; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm5[2,1,3,3]2302; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm1[2,1,1,0,4,5,6,7]2303; SSE-NEXT: pand %xmm0, %xmm52304; SSE-NEXT: por %xmm2, %xmm52305; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload2306; SSE-NEXT: # xmm2 = mem[2,2,3,3]2307; SSE-NEXT: movdqa %xmm0, %xmm12308; SSE-NEXT: pandn %xmm2, %xmm12309; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload2310; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload2311; SSE-NEXT: # xmm2 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]2312; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,1,3,3]2313; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm2[2,1,1,0,4,5,6,7]2314; SSE-NEXT: pand %xmm0, %xmm42315; SSE-NEXT: por %xmm1, %xmm42316; SSE-NEXT: pshufd $250, (%rsp), %xmm2 # 16-byte Folded Reload2317; SSE-NEXT: # xmm2 = mem[2,2,3,3]2318; SSE-NEXT: movdqa %xmm0, %xmm12319; SSE-NEXT: pandn %xmm2, %xmm12320; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload2321; SSE-NEXT: # xmm3 = xmm3[4],mem[4],xmm3[5],mem[5],xmm3[6],mem[6],xmm3[7],mem[7]2322; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,1,3,3]2323; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,1,0,4,5,6,7]2324; SSE-NEXT: pand %xmm0, %xmm22325; SSE-NEXT: por %xmm1, %xmm22326; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload2327; SSE-NEXT: # xmm3 = mem[2,2,3,3]2328; SSE-NEXT: movdqa %xmm0, %xmm12329; SSE-NEXT: pandn %xmm3, %xmm12330; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload2331; SSE-NEXT: # xmm14 = xmm14[4],mem[4],xmm14[5],mem[5],xmm14[6],mem[6],xmm14[7],mem[7]2332; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm14[2,1,3,3]2333; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm3[2,1,1,0,4,5,6,7]2334; SSE-NEXT: pand %xmm0, %xmm142335; SSE-NEXT: por %xmm1, %xmm142336; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload2337; SSE-NEXT: # xmm1 = mem[2,2,3,3]2338; SSE-NEXT: movdqa %xmm0, %xmm32339; SSE-NEXT: pandn %xmm1, %xmm32340; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload2341; SSE-NEXT: # xmm12 = xmm12[4],mem[4],xmm12[5],mem[5],xmm12[6],mem[6],xmm12[7],mem[7]2342; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm12[2,1,3,3]2343; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,1,0,4,5,6,7]2344; SSE-NEXT: pand %xmm0, %xmm12345; SSE-NEXT: por %xmm3, %xmm12346; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload2347; SSE-NEXT: # xmm3 = mem[2,2,3,3]2348; SSE-NEXT: movdqa %xmm0, %xmm122349; SSE-NEXT: pandn %xmm3, %xmm122350; SSE-NEXT: punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload2351; SSE-NEXT: # xmm9 = xmm9[4],mem[4],xmm9[5],mem[5],xmm9[6],mem[6],xmm9[7],mem[7]2352; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm9[2,1,3,3]2353; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[2,1,1,0,4,5,6,7]2354; SSE-NEXT: pand %xmm0, %xmm32355; SSE-NEXT: por %xmm12, %xmm32356; SSE-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7]2357; SSE-NEXT: pshufd $250, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload2358; SSE-NEXT: # xmm9 = mem[2,2,3,3]2359; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[2,1,3,3]2360; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[2,1,1,0,4,5,6,7]2361; SSE-NEXT: pand %xmm0, %xmm72362; SSE-NEXT: pandn %xmm9, %xmm02363; SSE-NEXT: por %xmm7, %xmm02364; SSE-NEXT: movdqa %xmm0, 368(%rcx)2365; SSE-NEXT: movdqa %xmm3, 320(%rcx)2366; SSE-NEXT: movdqa %xmm1, 272(%rcx)2367; SSE-NEXT: movdqa %xmm14, 224(%rcx)2368; SSE-NEXT: movdqa %xmm2, 176(%rcx)2369; SSE-NEXT: movdqa %xmm4, 128(%rcx)2370; SSE-NEXT: movdqa %xmm5, 80(%rcx)2371; SSE-NEXT: movdqa %xmm6, 32(%rcx)2372; SSE-NEXT: movdqa %xmm11, 352(%rcx)2373; SSE-NEXT: movdqa %xmm10, 336(%rcx)2374; SSE-NEXT: movdqa %xmm13, 304(%rcx)2375; SSE-NEXT: movdqa %xmm15, 288(%rcx)2376; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2377; SSE-NEXT: movaps %xmm0, 256(%rcx)2378; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2379; SSE-NEXT: movaps %xmm0, 240(%rcx)2380; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2381; SSE-NEXT: movaps %xmm0, 208(%rcx)2382; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2383; SSE-NEXT: movaps %xmm0, 192(%rcx)2384; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2385; SSE-NEXT: movaps %xmm0, 160(%rcx)2386; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2387; SSE-NEXT: movaps %xmm0, 144(%rcx)2388; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2389; SSE-NEXT: movaps %xmm0, 112(%rcx)2390; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2391; SSE-NEXT: movaps %xmm0, 96(%rcx)2392; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2393; SSE-NEXT: movaps %xmm0, 64(%rcx)2394; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2395; SSE-NEXT: movaps %xmm0, 48(%rcx)2396; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2397; SSE-NEXT: movaps %xmm0, 16(%rcx)2398; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2399; SSE-NEXT: movaps %xmm0, (%rcx)2400; SSE-NEXT: addq $328, %rsp # imm = 0x1482401; SSE-NEXT: retq2402;2403; AVX-LABEL: store_i16_stride3_vf64:2404; AVX: # %bb.0:2405; AVX-NEXT: subq $280, %rsp # imm = 0x1182406; AVX-NEXT: vmovdqa 80(%rdi), %xmm62407; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm6[1,1,2,2]2408; AVX-NEXT: vmovdqa 80(%rsi), %xmm122409; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm12[3,3,3,3,4,5,6,7]2410; AVX-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]2411; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]2412; AVX-NEXT: vmovdqa 80(%rdx), %xmm152413; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm15[1,1,2,2]2414; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]2415; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2416; AVX-NEXT: vmovdqa 64(%rdi), %xmm72417; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm7[1,1,2,2]2418; AVX-NEXT: vmovdqa 64(%rsi), %xmm12419; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2420; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,3,3,3,4,5,6,7]2421; AVX-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]2422; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]2423; AVX-NEXT: vmovdqa 64(%rdx), %xmm142424; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm14[1,1,2,2]2425; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]2426; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2427; AVX-NEXT: vmovdqa 32(%rdi), %xmm22428; AVX-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2429; AVX-NEXT: vmovdqa 48(%rdi), %xmm02430; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2431; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,2]2432; AVX-NEXT: vmovdqa 32(%rsi), %xmm32433; AVX-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2434; AVX-NEXT: vmovdqa 48(%rsi), %xmm12435; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2436; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,3,3,3,4,5,6,7]2437; AVX-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]2438; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]2439; AVX-NEXT: vmovdqa 48(%rdx), %xmm12440; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2441; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,2]2442; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]2443; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2444; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[1,1,2,2]2445; AVX-NEXT: vpshuflw {{.*#+}} xmm1 = xmm3[3,3,3,3,4,5,6,7]2446; AVX-NEXT: vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,4,4]2447; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]2448; AVX-NEXT: vmovdqa 32(%rdx), %xmm102449; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm10[1,1,2,2]2450; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]2451; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2452; AVX-NEXT: vmovdqa 112(%rsi), %xmm02453; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2454; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]2455; AVX-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]2456; AVX-NEXT: vmovdqa 112(%rdi), %xmm132457; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm13[1,1,2,2]2458; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]2459; AVX-NEXT: vmovdqa 112(%rdx), %xmm112460; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm11[1,1,2,2]2461; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]2462; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2463; AVX-NEXT: vmovdqa 96(%rsi), %xmm92464; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm9[3,3,3,3,4,5,6,7]2465; AVX-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]2466; AVX-NEXT: vmovdqa 96(%rdi), %xmm52467; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm5[1,1,2,2]2468; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]2469; AVX-NEXT: vmovdqa 96(%rdx), %xmm82470; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm8[1,1,2,2]2471; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]2472; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2473; AVX-NEXT: vmovdqa 16(%rsi), %xmm02474; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2475; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]2476; AVX-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]2477; AVX-NEXT: vmovdqa 16(%rdi), %xmm12478; AVX-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill2479; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,2]2480; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]2481; AVX-NEXT: vmovdqa 16(%rdx), %xmm12482; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2483; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,2]2484; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]2485; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2486; AVX-NEXT: vmovdqa (%rsi), %xmm02487; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2488; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[3,3,3,3,4,5,6,7]2489; AVX-NEXT: vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]2490; AVX-NEXT: vmovdqa (%rdi), %xmm12491; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2492; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,2]2493; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]2494; AVX-NEXT: vmovdqa (%rdx), %xmm12495; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2496; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,2]2497; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3],xmm0[4,5],xmm1[6],xmm0[7]2498; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2499; AVX-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm12[4],xmm6[4],xmm12[5],xmm6[5],xmm12[6],xmm6[6],xmm12[7],xmm6[7]2500; AVX-NEXT: vmovdqa {{.*#+}} xmm4 = [4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]2501; AVX-NEXT: vpshufb %xmm4, %xmm2, %xmm02502; AVX-NEXT: vmovdqa %xmm15, %xmm22503; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm15[2,2,3,3]2504; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm15[1],xmm0[2,3],xmm15[4],xmm0[5,6],xmm15[7]2505; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2506; AVX-NEXT: vmovdqa %xmm7, %xmm32507; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload2508; AVX-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3]2509; AVX-NEXT: vmovdqa {{.*#+}} xmm7 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]2510; AVX-NEXT: vpshufb %xmm7, %xmm15, %xmm152511; AVX-NEXT: vmovdqa %xmm14, %xmm02512; AVX-NEXT: vpshufd {{.*#+}} xmm14 = xmm14[0,0,0,0]2513; AVX-NEXT: vpblendw {{.*#+}} xmm14 = xmm15[0,1],xmm14[2],xmm15[3,4],xmm14[5],xmm15[6,7]2514; AVX-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2515; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm6[0],xmm12[0],xmm6[1],xmm12[1],xmm6[2],xmm12[2],xmm6[3],xmm12[3]2516; AVX-NEXT: vpshufb %xmm7, %xmm14, %xmm142517; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm2[0,0,0,0]2518; AVX-NEXT: vpblendw {{.*#+}} xmm12 = xmm14[0,1],xmm15[2],xmm14[3,4],xmm15[5],xmm14[6,7]2519; AVX-NEXT: vpunpckhwd {{.*#+}} xmm14 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]2520; AVX-NEXT: vpshufb %xmm4, %xmm14, %xmm142521; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm0[2,2,3,3]2522; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm14[0],xmm15[1],xmm14[2,3],xmm15[4],xmm14[5,6],xmm15[7]2523; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2524; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload2525; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload2526; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]2527; AVX-NEXT: vpshufb %xmm7, %xmm14, %xmm142528; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload2529; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm6[0,0,0,0]2530; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm14[0,1],xmm15[2],xmm14[3,4],xmm15[5],xmm14[6,7]2531; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2532; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2533; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload2534; AVX-NEXT: vpunpckhwd {{.*#+}} xmm14 = xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]2535; AVX-NEXT: vpshufb %xmm4, %xmm14, %xmm142536; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm10[2,2,3,3]2537; AVX-NEXT: vpblendw {{.*#+}} xmm14 = xmm14[0],xmm15[1],xmm14[2,3],xmm15[4],xmm14[5,6],xmm15[7]2538; AVX-NEXT: vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2539; AVX-NEXT: vpunpckhwd {{.*#+}} xmm14 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]2540; AVX-NEXT: vpshufb %xmm4, %xmm14, %xmm142541; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm6[2,2,3,3]2542; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm14[0],xmm15[1],xmm14[2,3],xmm15[4],xmm14[5,6],xmm15[7]2543; AVX-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2544; AVX-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]2545; AVX-NEXT: vpshufb %xmm7, %xmm14, %xmm142546; AVX-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[0,0,0,0]2547; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm14[0,1],xmm10[2],xmm14[3,4],xmm10[5],xmm14[6,7]2548; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2549; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2550; AVX-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3]2551; AVX-NEXT: vpshufb %xmm7, %xmm10, %xmm102552; AVX-NEXT: vpshufd {{.*#+}} xmm15 = xmm11[0,0,0,0]2553; AVX-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1],xmm15[2],xmm10[3,4],xmm15[5],xmm10[6,7]2554; AVX-NEXT: vpunpckhwd {{.*#+}} xmm15 = xmm9[4],xmm5[4],xmm9[5],xmm5[5],xmm9[6],xmm5[6],xmm9[7],xmm5[7]2555; AVX-NEXT: vpshufb %xmm4, %xmm15, %xmm152556; AVX-NEXT: vpshufd {{.*#+}} xmm14 = xmm8[2,2,3,3]2557; AVX-NEXT: vpblendw {{.*#+}} xmm14 = xmm15[0],xmm14[1],xmm15[2,3],xmm14[4],xmm15[5,6],xmm14[7]2558; AVX-NEXT: vpunpckhwd {{.*#+}} xmm13 = xmm0[4],xmm13[4],xmm0[5],xmm13[5],xmm0[6],xmm13[6],xmm0[7],xmm13[7]2559; AVX-NEXT: vmovdqa %xmm4, %xmm12560; AVX-NEXT: vpshufb %xmm4, %xmm13, %xmm132561; AVX-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[2,2,3,3]2562; AVX-NEXT: vpblendw {{.*#+}} xmm11 = xmm13[0],xmm11[1],xmm13[2,3],xmm11[4],xmm13[5,6],xmm11[7]2563; AVX-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3]2564; AVX-NEXT: vpshufb %xmm7, %xmm5, %xmm52565; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm8[0,0,0,0]2566; AVX-NEXT: vpblendw {{.*#+}} xmm4 = xmm5[0,1],xmm4[2],xmm5[3,4],xmm4[5],xmm5[6,7]2567; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2568; AVX-NEXT: vmovdqa (%rsp), %xmm8 # 16-byte Reload2569; AVX-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7]2570; AVX-NEXT: vpshufb %xmm1, %xmm5, %xmm52571; AVX-NEXT: vmovdqa %xmm1, %xmm152572; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload2573; AVX-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[2,2,3,3]2574; AVX-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0],xmm9[1],xmm5[2,3],xmm9[4],xmm5[5,6],xmm9[7]2575; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload2576; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload2577; AVX-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]2578; AVX-NEXT: vpshufb %xmm7, %xmm9, %xmm92579; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload2580; AVX-NEXT: vpshufd {{.*#+}} xmm13 = xmm6[0,0,0,0]2581; AVX-NEXT: vpblendw {{.*#+}} xmm9 = xmm9[0,1],xmm13[2],xmm9[3,4],xmm13[5],xmm9[6,7]2582; AVX-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]2583; AVX-NEXT: vpshufb %xmm7, %xmm8, %xmm02584; AVX-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[0,0,0,0]2585; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm7[2],xmm0[3,4],xmm7[5],xmm0[6,7]2586; AVX-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]2587; AVX-NEXT: vpshufb %xmm15, %xmm3, %xmm12588; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm6[2,2,3,3]2589; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3],xmm2[4],xmm1[5,6],xmm2[7]2590; AVX-NEXT: vmovdqa %xmm1, 32(%rcx)2591; AVX-NEXT: vmovdqa %xmm0, 48(%rcx)2592; AVX-NEXT: vmovdqa %xmm9, (%rcx)2593; AVX-NEXT: vmovdqa %xmm5, 80(%rcx)2594; AVX-NEXT: vmovdqa %xmm4, 288(%rcx)2595; AVX-NEXT: vmovdqa %xmm11, 368(%rcx)2596; AVX-NEXT: vmovdqa %xmm14, 320(%rcx)2597; AVX-NEXT: vmovdqa %xmm10, 336(%rcx)2598; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2599; AVX-NEXT: vmovaps %xmm0, 96(%rcx)2600; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2601; AVX-NEXT: vmovaps %xmm0, 176(%rcx)2602; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2603; AVX-NEXT: vmovaps %xmm0, 128(%rcx)2604; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2605; AVX-NEXT: vmovaps %xmm0, 144(%rcx)2606; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2607; AVX-NEXT: vmovaps %xmm0, 224(%rcx)2608; AVX-NEXT: vmovdqa %xmm12, 240(%rcx)2609; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2610; AVX-NEXT: vmovaps %xmm0, 192(%rcx)2611; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2612; AVX-NEXT: vmovaps %xmm0, 272(%rcx)2613; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2614; AVX-NEXT: vmovaps %xmm0, 16(%rcx)2615; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2616; AVX-NEXT: vmovaps %xmm0, 64(%rcx)2617; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2618; AVX-NEXT: vmovaps %xmm0, 304(%rcx)2619; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2620; AVX-NEXT: vmovaps %xmm0, 352(%rcx)2621; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2622; AVX-NEXT: vmovaps %xmm0, 112(%rcx)2623; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2624; AVX-NEXT: vmovaps %xmm0, 160(%rcx)2625; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2626; AVX-NEXT: vmovaps %xmm0, 208(%rcx)2627; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2628; AVX-NEXT: vmovaps %xmm0, 256(%rcx)2629; AVX-NEXT: addq $280, %rsp # imm = 0x1182630; AVX-NEXT: retq2631;2632; AVX2-LABEL: store_i16_stride3_vf64:2633; AVX2: # %bb.0:2634; AVX2-NEXT: vmovdqa (%rdx), %ymm42635; AVX2-NEXT: vmovdqa 32(%rdx), %ymm02636; AVX2-NEXT: vmovdqa 64(%rdx), %ymm32637; AVX2-NEXT: vmovdqa 96(%rdx), %ymm12638; AVX2-NEXT: vmovdqa 80(%rdi), %xmm22639; AVX2-NEXT: vmovdqa 80(%rsi), %xmm52640; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]2641; AVX2-NEXT: vmovdqa {{.*#+}} xmm7 = [4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]2642; AVX2-NEXT: vpshufb %xmm7, %xmm6, %xmm62643; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[1,1,2,2]2644; AVX2-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[3,3,3,3,4,5,6,7]2645; AVX2-NEXT: vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,4,4,4]2646; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm5[2],xmm2[3,4],xmm5[5],xmm2[6,7]2647; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm2, %ymm22648; AVX2-NEXT: vpmovsxbd {{.*#+}} ymm8 = [5,5,0,6,6,0,7,7]2649; AVX2-NEXT: vpermd %ymm3, %ymm8, %ymm52650; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm9 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]2651; AVX2-NEXT: vpblendvb %ymm9, %ymm2, %ymm5, %ymm22652; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2653; AVX2-NEXT: vmovdqa 48(%rdi), %xmm52654; AVX2-NEXT: vmovdqa 48(%rsi), %xmm62655; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm10 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]2656; AVX2-NEXT: vpshufb %xmm7, %xmm10, %xmm102657; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,2]2658; AVX2-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[3,3,3,3,4,5,6,7]2659; AVX2-NEXT: vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,4,4]2660; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm6[2],xmm5[3,4],xmm6[5],xmm5[6,7]2661; AVX2-NEXT: vinserti128 $1, %xmm10, %ymm5, %ymm52662; AVX2-NEXT: vpermd %ymm0, %ymm8, %ymm62663; AVX2-NEXT: vpblendvb %ymm9, %ymm5, %ymm6, %ymm22664; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2665; AVX2-NEXT: vmovdqa 112(%rdi), %xmm62666; AVX2-NEXT: vmovdqa 112(%rsi), %xmm102667; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm11 = xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]2668; AVX2-NEXT: vpshufb %xmm7, %xmm11, %xmm112669; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[1,1,2,2]2670; AVX2-NEXT: vpshuflw {{.*#+}} xmm10 = xmm10[3,3,3,3,4,5,6,7]2671; AVX2-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,4,4,4]2672; AVX2-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm10[2],xmm6[3,4],xmm10[5],xmm6[6,7]2673; AVX2-NEXT: vinserti128 $1, %xmm11, %ymm6, %ymm62674; AVX2-NEXT: vpermd %ymm1, %ymm8, %ymm102675; AVX2-NEXT: vpblendvb %ymm9, %ymm6, %ymm10, %ymm22676; AVX2-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2677; AVX2-NEXT: vmovdqa (%rdi), %xmm102678; AVX2-NEXT: vmovdqa 16(%rdi), %xmm112679; AVX2-NEXT: vmovdqa (%rsi), %xmm122680; AVX2-NEXT: vmovdqa 16(%rsi), %xmm132681; AVX2-NEXT: vpunpckhwd {{.*#+}} xmm14 = xmm13[4],xmm11[4],xmm13[5],xmm11[5],xmm13[6],xmm11[6],xmm13[7],xmm11[7]2682; AVX2-NEXT: vpshufb %xmm7, %xmm14, %xmm72683; AVX2-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[1,1,2,2]2684; AVX2-NEXT: vpshuflw {{.*#+}} xmm13 = xmm13[3,3,3,3,4,5,6,7]2685; AVX2-NEXT: vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,4,4,4,4]2686; AVX2-NEXT: vpblendw {{.*#+}} xmm11 = xmm11[0,1],xmm13[2],xmm11[3,4],xmm13[5],xmm11[6,7]2687; AVX2-NEXT: vinserti128 $1, %xmm7, %ymm11, %ymm72688; AVX2-NEXT: vpermd %ymm4, %ymm8, %ymm82689; AVX2-NEXT: vpblendvb %ymm9, %ymm7, %ymm8, %ymm72690; AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm10[1,1,2,2]2691; AVX2-NEXT: vpshuflw {{.*#+}} xmm9 = xmm12[3,3,3,3,4,5,6,7]2692; AVX2-NEXT: vpshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,4,4,4]2693; AVX2-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0,1],xmm9[2],xmm8[3,4],xmm9[5],xmm8[6,7]2694; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm12[0],xmm10[1],xmm12[1],xmm10[2],xmm12[2],xmm10[3],xmm12[3]2695; AVX2-NEXT: vmovdqa {{.*#+}} xmm11 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]2696; AVX2-NEXT: vpshufb %xmm11, %xmm9, %xmm92697; AVX2-NEXT: vinserti128 $1, %xmm8, %ymm9, %ymm82698; AVX2-NEXT: vpmovsxbd {{.*#+}} ymm12 = [0,0,0,0,1,1,0,2]2699; AVX2-NEXT: vpermd %ymm4, %ymm12, %ymm92700; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm13 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]2701; AVX2-NEXT: vpblendvb %ymm13, %ymm8, %ymm9, %ymm82702; AVX2-NEXT: vmovdqa 64(%rdi), %xmm92703; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm9[1,1,2,2]2704; AVX2-NEXT: vmovdqa 64(%rsi), %xmm142705; AVX2-NEXT: vpshuflw {{.*#+}} xmm15 = xmm14[3,3,3,3,4,5,6,7]2706; AVX2-NEXT: vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,4,4,4]2707; AVX2-NEXT: vpblendw {{.*#+}} xmm10 = xmm10[0,1],xmm15[2],xmm10[3,4],xmm15[5],xmm10[6,7]2708; AVX2-NEXT: vmovdqa 32(%rdi), %xmm152709; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm14[0],xmm9[1],xmm14[1],xmm9[2],xmm14[2],xmm9[3],xmm14[3]2710; AVX2-NEXT: vmovdqa 32(%rsi), %xmm142711; AVX2-NEXT: vpshufb %xmm11, %xmm9, %xmm92712; AVX2-NEXT: vinserti128 $1, %xmm10, %ymm9, %ymm92713; AVX2-NEXT: vpermd %ymm3, %ymm12, %ymm102714; AVX2-NEXT: vpblendvb %ymm13, %ymm9, %ymm10, %ymm92715; AVX2-NEXT: vpshuflw {{.*#+}} xmm10 = xmm14[3,3,3,3,4,5,6,7]2716; AVX2-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,4,4,4]2717; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm15[1,1,2,2]2718; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm10[2],xmm2[3,4],xmm10[5],xmm2[6,7]2719; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]2720; AVX2-NEXT: vpshufb %xmm11, %xmm10, %xmm102721; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm10, %ymm22722; AVX2-NEXT: vpermd %ymm0, %ymm12, %ymm102723; AVX2-NEXT: vpblendvb %ymm13, %ymm2, %ymm10, %ymm102724; AVX2-NEXT: vmovdqa 96(%rsi), %xmm22725; AVX2-NEXT: vpshuflw {{.*#+}} xmm14 = xmm2[3,3,3,3,4,5,6,7]2726; AVX2-NEXT: vpshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,4,4,4,4]2727; AVX2-NEXT: vmovdqa 96(%rdi), %xmm152728; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm15[1,1,2,2]2729; AVX2-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm14[2],xmm5[3,4],xmm14[5],xmm5[6,7]2730; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm15[0],xmm2[0],xmm15[1],xmm2[1],xmm15[2],xmm2[2],xmm15[3],xmm2[3]2731; AVX2-NEXT: vpshufb %xmm11, %xmm2, %xmm22732; AVX2-NEXT: vmovdqa (%rsi), %ymm142733; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm2, %ymm22734; AVX2-NEXT: vpermd %ymm1, %ymm12, %ymm52735; AVX2-NEXT: vpblendvb %ymm13, %ymm2, %ymm5, %ymm112736; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [10,11,0,1,12,13,12,13,2,3,2,3,14,15,4,5,26,27,16,17,28,29,28,29,18,19,18,19,30,31,20,21]2737; AVX2-NEXT: vpshufb %ymm2, %ymm14, %ymm52738; AVX2-NEXT: vpmovsxbd {{.*#+}} ymm12 = [0,3,3,0,4,4,0,5]2739; AVX2-NEXT: vpermd (%rdi), %ymm12, %ymm132740; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm14 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]2741; AVX2-NEXT: vpblendvb %ymm14, %ymm5, %ymm13, %ymm52742; AVX2-NEXT: vmovdqa 64(%rsi), %ymm132743; AVX2-NEXT: vpshufb %ymm2, %ymm13, %ymm132744; AVX2-NEXT: vpermd 64(%rdi), %ymm12, %ymm152745; AVX2-NEXT: vpblendvb %ymm14, %ymm13, %ymm15, %ymm132746; AVX2-NEXT: vmovdqa 32(%rsi), %ymm152747; AVX2-NEXT: vpshufb %ymm2, %ymm15, %ymm152748; AVX2-NEXT: vpermd 32(%rdi), %ymm12, %ymm62749; AVX2-NEXT: vpblendvb %ymm14, %ymm15, %ymm6, %ymm62750; AVX2-NEXT: vmovdqa 96(%rsi), %ymm152751; AVX2-NEXT: vpshufb %ymm2, %ymm15, %ymm22752; AVX2-NEXT: vpermd 96(%rdi), %ymm12, %ymm122753; AVX2-NEXT: vpblendvb %ymm14, %ymm2, %ymm12, %ymm22754; AVX2-NEXT: vpmovsxbd {{.*#+}} ymm12 = [2,0,3,3,0,4,4,0]2755; AVX2-NEXT: vpermd %ymm4, %ymm12, %ymm42756; AVX2-NEXT: vpmovsxbw {{.*#+}} ymm14 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]2757; AVX2-NEXT: vpblendvb %ymm14, %ymm5, %ymm4, %ymm42758; AVX2-NEXT: vpermd %ymm3, %ymm12, %ymm32759; AVX2-NEXT: vpblendvb %ymm14, %ymm13, %ymm3, %ymm32760; AVX2-NEXT: vpermd %ymm0, %ymm12, %ymm02761; AVX2-NEXT: vpblendvb %ymm14, %ymm6, %ymm0, %ymm02762; AVX2-NEXT: vpermd %ymm1, %ymm12, %ymm12763; AVX2-NEXT: vpblendvb %ymm14, %ymm2, %ymm1, %ymm12764; AVX2-NEXT: vmovdqa %ymm1, 320(%rcx)2765; AVX2-NEXT: vmovdqa %ymm0, 128(%rcx)2766; AVX2-NEXT: vmovdqa %ymm3, 224(%rcx)2767; AVX2-NEXT: vmovdqa %ymm4, 32(%rcx)2768; AVX2-NEXT: vmovdqa %ymm11, 288(%rcx)2769; AVX2-NEXT: vmovdqa %ymm10, 96(%rcx)2770; AVX2-NEXT: vmovdqa %ymm9, 192(%rcx)2771; AVX2-NEXT: vmovdqa %ymm8, (%rcx)2772; AVX2-NEXT: vmovdqa %ymm7, 64(%rcx)2773; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2774; AVX2-NEXT: vmovaps %ymm0, 352(%rcx)2775; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2776; AVX2-NEXT: vmovaps %ymm0, 160(%rcx)2777; AVX2-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2778; AVX2-NEXT: vmovaps %ymm0, 256(%rcx)2779; AVX2-NEXT: vzeroupper2780; AVX2-NEXT: retq2781;2782; AVX2-FP-LABEL: store_i16_stride3_vf64:2783; AVX2-FP: # %bb.0:2784; AVX2-FP-NEXT: vmovdqa (%rdx), %ymm22785; AVX2-FP-NEXT: vmovdqa 32(%rdx), %ymm02786; AVX2-FP-NEXT: vmovdqa 64(%rdx), %ymm12787; AVX2-FP-NEXT: vmovdqa (%rsi), %xmm32788; AVX2-FP-NEXT: vpbroadcastd {{.*#+}} xmm6 = [6,7,8,9,6,7,8,9,6,7,8,9,6,7,8,9]2789; AVX2-FP-NEXT: vpshufb %xmm6, %xmm3, %xmm42790; AVX2-FP-NEXT: vmovdqa (%rdi), %xmm52791; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,2,2]2792; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm7[0,1],xmm4[2],xmm7[3,4],xmm4[5],xmm7[6,7]2793; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]2794; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm12 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]2795; AVX2-FP-NEXT: vpshufb %xmm12, %xmm3, %xmm32796; AVX2-FP-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm32797; AVX2-FP-NEXT: vpmovsxbd {{.*#+}} ymm13 = [0,0,0,0,1,1,0,2]2798; AVX2-FP-NEXT: vpermd %ymm2, %ymm13, %ymm42799; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm14 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]2800; AVX2-FP-NEXT: vpblendvb %ymm14, %ymm3, %ymm4, %ymm32801; AVX2-FP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2802; AVX2-FP-NEXT: vmovdqa 80(%rdi), %xmm42803; AVX2-FP-NEXT: vmovdqa 80(%rsi), %xmm52804; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]2805; AVX2-FP-NEXT: vmovdqa {{.*#+}} xmm9 = [4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]2806; AVX2-FP-NEXT: vpshufb %xmm9, %xmm7, %xmm72807; AVX2-FP-NEXT: vpshufb %xmm6, %xmm5, %xmm52808; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,1,2,2]2809; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]2810; AVX2-FP-NEXT: vinserti128 $1, %xmm7, %ymm4, %ymm42811; AVX2-FP-NEXT: vpmovsxbd {{.*#+}} ymm10 = [5,5,0,6,6,0,7,7]2812; AVX2-FP-NEXT: vpermd %ymm1, %ymm10, %ymm52813; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm11 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]2814; AVX2-FP-NEXT: vpblendvb %ymm11, %ymm4, %ymm5, %ymm32815; AVX2-FP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2816; AVX2-FP-NEXT: vmovdqa 64(%rsi), %xmm52817; AVX2-FP-NEXT: vpshufb %xmm6, %xmm5, %xmm72818; AVX2-FP-NEXT: vmovdqa 64(%rdi), %xmm82819; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm15 = xmm8[1,1,2,2]2820; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm7 = xmm15[0,1],xmm7[2],xmm15[3,4],xmm7[5],xmm15[6,7]2821; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]2822; AVX2-FP-NEXT: vpshufb %xmm12, %xmm5, %xmm52823; AVX2-FP-NEXT: vinserti128 $1, %xmm7, %ymm5, %ymm52824; AVX2-FP-NEXT: vpermd %ymm1, %ymm13, %ymm72825; AVX2-FP-NEXT: vpblendvb %ymm14, %ymm5, %ymm7, %ymm32826; AVX2-FP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2827; AVX2-FP-NEXT: vmovdqa 48(%rdi), %xmm72828; AVX2-FP-NEXT: vmovdqa 48(%rsi), %xmm82829; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm15 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]2830; AVX2-FP-NEXT: vpshufb %xmm6, %xmm8, %xmm82831; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[1,1,2,2]2832; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm8[2],xmm7[3,4],xmm8[5],xmm7[6,7]2833; AVX2-FP-NEXT: vmovdqa 32(%rsi), %xmm82834; AVX2-FP-NEXT: vpshufb %xmm9, %xmm15, %xmm152835; AVX2-FP-NEXT: vinserti128 $1, %xmm15, %ymm7, %ymm72836; AVX2-FP-NEXT: vmovdqa %ymm0, %ymm52837; AVX2-FP-NEXT: vpermd %ymm0, %ymm10, %ymm152838; AVX2-FP-NEXT: vpblendvb %ymm11, %ymm7, %ymm15, %ymm72839; AVX2-FP-NEXT: vmovdqa 32(%rdi), %xmm152840; AVX2-FP-NEXT: vpshufb %xmm6, %xmm8, %xmm02841; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm3 = xmm15[1,1,2,2]2842; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1],xmm0[2],xmm3[3,4],xmm0[5],xmm3[6,7]2843; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3]2844; AVX2-FP-NEXT: vpshufb %xmm12, %xmm3, %xmm32845; AVX2-FP-NEXT: vinserti128 $1, %xmm0, %ymm3, %ymm02846; AVX2-FP-NEXT: vpermd %ymm5, %ymm13, %ymm32847; AVX2-FP-NEXT: vpblendvb %ymm14, %ymm0, %ymm3, %ymm82848; AVX2-FP-NEXT: vmovdqa 96(%rsi), %xmm02849; AVX2-FP-NEXT: vpshufb %xmm6, %xmm0, %xmm32850; AVX2-FP-NEXT: vmovdqa 96(%rdi), %xmm152851; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm4 = xmm15[1,1,2,2]2852; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm3[2],xmm4[3,4],xmm3[5],xmm4[6,7]2853; AVX2-FP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]2854; AVX2-FP-NEXT: vpshufb %xmm12, %xmm0, %xmm02855; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm02856; AVX2-FP-NEXT: vmovdqa 96(%rdx), %ymm152857; AVX2-FP-NEXT: vpermd %ymm15, %ymm13, %ymm32858; AVX2-FP-NEXT: vpblendvb %ymm14, %ymm0, %ymm3, %ymm122859; AVX2-FP-NEXT: vmovdqa 112(%rdi), %xmm02860; AVX2-FP-NEXT: vmovdqa 112(%rsi), %xmm32861; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]2862; AVX2-FP-NEXT: vpshufb %xmm6, %xmm3, %xmm32863; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,2]2864; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2],xmm0[3,4],xmm3[5],xmm0[6,7]2865; AVX2-FP-NEXT: vpshufb %xmm9, %xmm4, %xmm32866; AVX2-FP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm02867; AVX2-FP-NEXT: vpermd %ymm15, %ymm10, %ymm32868; AVX2-FP-NEXT: vpblendvb %ymm11, %ymm0, %ymm3, %ymm132869; AVX2-FP-NEXT: vmovdqa 16(%rsi), %xmm02870; AVX2-FP-NEXT: vmovdqa 16(%rdi), %xmm32871; AVX2-FP-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]2872; AVX2-FP-NEXT: vpshufb %xmm9, %xmm4, %xmm42873; AVX2-FP-NEXT: vpshufb %xmm6, %xmm0, %xmm02874; AVX2-FP-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,1,2,2]2875; AVX2-FP-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1],xmm0[2],xmm3[3,4],xmm0[5],xmm3[6,7]2876; AVX2-FP-NEXT: vmovdqa (%rsi), %ymm32877; AVX2-FP-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm02878; AVX2-FP-NEXT: vpermd %ymm2, %ymm10, %ymm42879; AVX2-FP-NEXT: vpblendvb %ymm11, %ymm0, %ymm4, %ymm62880; AVX2-FP-NEXT: vmovdqa {{.*#+}} ymm0 = [10,11,0,1,12,13,12,13,2,3,2,3,14,15,4,5,26,27,16,17,28,29,28,29,18,19,18,19,30,31,20,21]2881; AVX2-FP-NEXT: vpshufb %ymm0, %ymm3, %ymm32882; AVX2-FP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,3,3,0,4,4,0,5]2883; AVX2-FP-NEXT: vpermd (%rdi), %ymm4, %ymm92884; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm10 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]2885; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm3, %ymm9, %ymm32886; AVX2-FP-NEXT: vmovdqa 64(%rsi), %ymm92887; AVX2-FP-NEXT: vpshufb %ymm0, %ymm9, %ymm92888; AVX2-FP-NEXT: vpermd 64(%rdi), %ymm4, %ymm112889; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm9, %ymm11, %ymm92890; AVX2-FP-NEXT: vmovdqa 32(%rsi), %ymm112891; AVX2-FP-NEXT: vpshufb %ymm0, %ymm11, %ymm112892; AVX2-FP-NEXT: vpermd 32(%rdi), %ymm4, %ymm142893; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm11, %ymm14, %ymm112894; AVX2-FP-NEXT: vmovdqa 96(%rsi), %ymm142895; AVX2-FP-NEXT: vpshufb %ymm0, %ymm14, %ymm02896; AVX2-FP-NEXT: vpermd 96(%rdi), %ymm4, %ymm42897; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm0, %ymm4, %ymm02898; AVX2-FP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [2,0,3,3,0,4,4,0]2899; AVX2-FP-NEXT: vpermd %ymm2, %ymm4, %ymm22900; AVX2-FP-NEXT: vpmovsxbw {{.*#+}} ymm10 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]2901; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm3, %ymm2, %ymm22902; AVX2-FP-NEXT: vpermd %ymm1, %ymm4, %ymm12903; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm9, %ymm1, %ymm12904; AVX2-FP-NEXT: vpermd %ymm5, %ymm4, %ymm32905; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm11, %ymm3, %ymm32906; AVX2-FP-NEXT: vpermd %ymm15, %ymm4, %ymm42907; AVX2-FP-NEXT: vpblendvb %ymm10, %ymm0, %ymm4, %ymm02908; AVX2-FP-NEXT: vmovdqa %ymm0, 320(%rcx)2909; AVX2-FP-NEXT: vmovdqa %ymm3, 128(%rcx)2910; AVX2-FP-NEXT: vmovdqa %ymm1, 224(%rcx)2911; AVX2-FP-NEXT: vmovdqa %ymm2, 32(%rcx)2912; AVX2-FP-NEXT: vmovdqa %ymm6, 64(%rcx)2913; AVX2-FP-NEXT: vmovdqa %ymm12, 288(%rcx)2914; AVX2-FP-NEXT: vmovdqa %ymm13, 352(%rcx)2915; AVX2-FP-NEXT: vmovdqa %ymm8, 96(%rcx)2916; AVX2-FP-NEXT: vmovdqa %ymm7, 160(%rcx)2917; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2918; AVX2-FP-NEXT: vmovaps %ymm0, 192(%rcx)2919; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2920; AVX2-FP-NEXT: vmovaps %ymm0, 256(%rcx)2921; AVX2-FP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2922; AVX2-FP-NEXT: vmovaps %ymm0, (%rcx)2923; AVX2-FP-NEXT: vzeroupper2924; AVX2-FP-NEXT: retq2925;2926; AVX2-FCP-LABEL: store_i16_stride3_vf64:2927; AVX2-FCP: # %bb.0:2928; AVX2-FCP-NEXT: vmovdqa (%rdx), %ymm22929; AVX2-FCP-NEXT: vmovdqa 32(%rdx), %ymm02930; AVX2-FCP-NEXT: vmovdqa 64(%rdx), %ymm12931; AVX2-FCP-NEXT: vmovdqa (%rsi), %xmm32932; AVX2-FCP-NEXT: vpbroadcastd {{.*#+}} xmm6 = [6,7,8,9,6,7,8,9,6,7,8,9,6,7,8,9]2933; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm3, %xmm42934; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm52935; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[1,1,2,2]2936; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm7[0,1],xmm4[2],xmm7[3,4],xmm4[5],xmm7[6,7]2937; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]2938; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm12 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]2939; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm3, %xmm32940; AVX2-FCP-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm32941; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm13 = [0,0,0,0,1,1,0,2]2942; AVX2-FCP-NEXT: vpermd %ymm2, %ymm13, %ymm42943; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm14 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]2944; AVX2-FCP-NEXT: vpblendvb %ymm14, %ymm3, %ymm4, %ymm32945; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2946; AVX2-FCP-NEXT: vmovdqa 80(%rdi), %xmm42947; AVX2-FCP-NEXT: vmovdqa 80(%rsi), %xmm52948; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]2949; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm9 = [4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]2950; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm7, %xmm72951; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm5, %xmm52952; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,1,2,2]2953; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]2954; AVX2-FCP-NEXT: vinserti128 $1, %xmm7, %ymm4, %ymm42955; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm10 = [5,5,0,6,6,0,7,7]2956; AVX2-FCP-NEXT: vpermd %ymm1, %ymm10, %ymm52957; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm11 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]2958; AVX2-FCP-NEXT: vpblendvb %ymm11, %ymm4, %ymm5, %ymm32959; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2960; AVX2-FCP-NEXT: vmovdqa 64(%rsi), %xmm52961; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm5, %xmm72962; AVX2-FCP-NEXT: vmovdqa 64(%rdi), %xmm82963; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm15 = xmm8[1,1,2,2]2964; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm15[0,1],xmm7[2],xmm15[3,4],xmm7[5],xmm15[6,7]2965; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]2966; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm5, %xmm52967; AVX2-FCP-NEXT: vinserti128 $1, %xmm7, %ymm5, %ymm52968; AVX2-FCP-NEXT: vpermd %ymm1, %ymm13, %ymm72969; AVX2-FCP-NEXT: vpblendvb %ymm14, %ymm5, %ymm7, %ymm32970; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2971; AVX2-FCP-NEXT: vmovdqa 48(%rdi), %xmm72972; AVX2-FCP-NEXT: vmovdqa 48(%rsi), %xmm82973; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm15 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]2974; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm8, %xmm82975; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[1,1,2,2]2976; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0,1],xmm8[2],xmm7[3,4],xmm8[5],xmm7[6,7]2977; AVX2-FCP-NEXT: vmovdqa 32(%rsi), %xmm82978; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm15, %xmm152979; AVX2-FCP-NEXT: vinserti128 $1, %xmm15, %ymm7, %ymm72980; AVX2-FCP-NEXT: vmovdqa %ymm0, %ymm52981; AVX2-FCP-NEXT: vpermd %ymm0, %ymm10, %ymm152982; AVX2-FCP-NEXT: vpblendvb %ymm11, %ymm7, %ymm15, %ymm72983; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm152984; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm8, %xmm02985; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm3 = xmm15[1,1,2,2]2986; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1],xmm0[2],xmm3[3,4],xmm0[5],xmm3[6,7]2987; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3]2988; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm3, %xmm32989; AVX2-FCP-NEXT: vinserti128 $1, %xmm0, %ymm3, %ymm02990; AVX2-FCP-NEXT: vpermd %ymm5, %ymm13, %ymm32991; AVX2-FCP-NEXT: vpblendvb %ymm14, %ymm0, %ymm3, %ymm82992; AVX2-FCP-NEXT: vmovdqa 96(%rsi), %xmm02993; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm0, %xmm32994; AVX2-FCP-NEXT: vmovdqa 96(%rdi), %xmm152995; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm4 = xmm15[1,1,2,2]2996; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm3[2],xmm4[3,4],xmm3[5],xmm4[6,7]2997; AVX2-FCP-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]2998; AVX2-FCP-NEXT: vpshufb %xmm12, %xmm0, %xmm02999; AVX2-FCP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm03000; AVX2-FCP-NEXT: vmovdqa 96(%rdx), %ymm153001; AVX2-FCP-NEXT: vpermd %ymm15, %ymm13, %ymm33002; AVX2-FCP-NEXT: vpblendvb %ymm14, %ymm0, %ymm3, %ymm123003; AVX2-FCP-NEXT: vmovdqa 112(%rdi), %xmm03004; AVX2-FCP-NEXT: vmovdqa 112(%rsi), %xmm33005; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]3006; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm3, %xmm33007; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,2]3008; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2],xmm0[3,4],xmm3[5],xmm0[6,7]3009; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm4, %xmm33010; AVX2-FCP-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm03011; AVX2-FCP-NEXT: vpermd %ymm15, %ymm10, %ymm33012; AVX2-FCP-NEXT: vpblendvb %ymm11, %ymm0, %ymm3, %ymm133013; AVX2-FCP-NEXT: vmovdqa 16(%rsi), %xmm03014; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm33015; AVX2-FCP-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]3016; AVX2-FCP-NEXT: vpshufb %xmm9, %xmm4, %xmm43017; AVX2-FCP-NEXT: vpshufb %xmm6, %xmm0, %xmm03018; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[1,1,2,2]3019; AVX2-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1],xmm0[2],xmm3[3,4],xmm0[5],xmm3[6,7]3020; AVX2-FCP-NEXT: vmovdqa (%rsi), %ymm33021; AVX2-FCP-NEXT: vinserti128 $1, %xmm4, %ymm0, %ymm03022; AVX2-FCP-NEXT: vpermd %ymm2, %ymm10, %ymm43023; AVX2-FCP-NEXT: vpblendvb %ymm11, %ymm0, %ymm4, %ymm63024; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm0 = [10,11,0,1,12,13,12,13,2,3,2,3,14,15,4,5,26,27,16,17,28,29,28,29,18,19,18,19,30,31,20,21]3025; AVX2-FCP-NEXT: vpshufb %ymm0, %ymm3, %ymm33026; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [0,3,3,0,4,4,0,5]3027; AVX2-FCP-NEXT: vpermd (%rdi), %ymm4, %ymm93028; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm10 = [65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535,0,0,65535]3029; AVX2-FCP-NEXT: vpblendvb %ymm10, %ymm3, %ymm9, %ymm33030; AVX2-FCP-NEXT: vmovdqa 64(%rsi), %ymm93031; AVX2-FCP-NEXT: vpshufb %ymm0, %ymm9, %ymm93032; AVX2-FCP-NEXT: vpermd 64(%rdi), %ymm4, %ymm113033; AVX2-FCP-NEXT: vpblendvb %ymm10, %ymm9, %ymm11, %ymm93034; AVX2-FCP-NEXT: vmovdqa 32(%rsi), %ymm113035; AVX2-FCP-NEXT: vpshufb %ymm0, %ymm11, %ymm113036; AVX2-FCP-NEXT: vpermd 32(%rdi), %ymm4, %ymm143037; AVX2-FCP-NEXT: vpblendvb %ymm10, %ymm11, %ymm14, %ymm113038; AVX2-FCP-NEXT: vmovdqa 96(%rsi), %ymm143039; AVX2-FCP-NEXT: vpshufb %ymm0, %ymm14, %ymm03040; AVX2-FCP-NEXT: vpermd 96(%rdi), %ymm4, %ymm43041; AVX2-FCP-NEXT: vpblendvb %ymm10, %ymm0, %ymm4, %ymm03042; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm4 = [2,0,3,3,0,4,4,0]3043; AVX2-FCP-NEXT: vpermd %ymm2, %ymm4, %ymm23044; AVX2-FCP-NEXT: vpmovsxbw {{.*#+}} ymm10 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]3045; AVX2-FCP-NEXT: vpblendvb %ymm10, %ymm3, %ymm2, %ymm23046; AVX2-FCP-NEXT: vpermd %ymm1, %ymm4, %ymm13047; AVX2-FCP-NEXT: vpblendvb %ymm10, %ymm9, %ymm1, %ymm13048; AVX2-FCP-NEXT: vpermd %ymm5, %ymm4, %ymm33049; AVX2-FCP-NEXT: vpblendvb %ymm10, %ymm11, %ymm3, %ymm33050; AVX2-FCP-NEXT: vpermd %ymm15, %ymm4, %ymm43051; AVX2-FCP-NEXT: vpblendvb %ymm10, %ymm0, %ymm4, %ymm03052; AVX2-FCP-NEXT: vmovdqa %ymm0, 320(%rcx)3053; AVX2-FCP-NEXT: vmovdqa %ymm3, 128(%rcx)3054; AVX2-FCP-NEXT: vmovdqa %ymm1, 224(%rcx)3055; AVX2-FCP-NEXT: vmovdqa %ymm2, 32(%rcx)3056; AVX2-FCP-NEXT: vmovdqa %ymm6, 64(%rcx)3057; AVX2-FCP-NEXT: vmovdqa %ymm12, 288(%rcx)3058; AVX2-FCP-NEXT: vmovdqa %ymm13, 352(%rcx)3059; AVX2-FCP-NEXT: vmovdqa %ymm8, 96(%rcx)3060; AVX2-FCP-NEXT: vmovdqa %ymm7, 160(%rcx)3061; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3062; AVX2-FCP-NEXT: vmovaps %ymm0, 192(%rcx)3063; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3064; AVX2-FCP-NEXT: vmovaps %ymm0, 256(%rcx)3065; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3066; AVX2-FCP-NEXT: vmovaps %ymm0, (%rcx)3067; AVX2-FCP-NEXT: vzeroupper3068; AVX2-FCP-NEXT: retq3069;3070; AVX512-LABEL: store_i16_stride3_vf64:3071; AVX512: # %bb.0:3072; AVX512-NEXT: vmovdqa (%rdi), %ymm03073; AVX512-NEXT: vmovdqa {{.*#+}} ymm6 = [128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128,20,21,128,128]3074; AVX512-NEXT: vpshufb %ymm6, %ymm0, %ymm03075; AVX512-NEXT: vmovdqa (%rsi), %ymm13076; AVX512-NEXT: vmovdqa {{.*#+}} ymm7 = [10,11,0,1,128,128,12,13,2,3,128,128,14,15,4,5,128,128,16,17,28,29,128,128,18,19,18,19,128,128,20,21]3077; AVX512-NEXT: vpshufb %ymm7, %ymm1, %ymm13078; AVX512-NEXT: vpor %ymm0, %ymm1, %ymm33079; AVX512-NEXT: vmovdqa (%rsi), %xmm53080; AVX512-NEXT: vmovdqa64 16(%rsi), %xmm203081; AVX512-NEXT: vmovdqa64 32(%rsi), %xmm243082; AVX512-NEXT: vprold $16, %xmm5, %xmm83083; AVX512-NEXT: vmovdqa (%rdi), %xmm93084; AVX512-NEXT: vmovdqa64 16(%rdi), %xmm213085; AVX512-NEXT: vmovdqa 32(%rdi), %xmm43086; AVX512-NEXT: vpshufd {{.*#+}} xmm10 = xmm9[1,1,2,2]3087; AVX512-NEXT: vpblendw {{.*#+}} xmm8 = xmm10[0,1],xmm8[2],xmm10[3,4],xmm8[5],xmm10[6,7]3088; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]3089; AVX512-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]3090; AVX512-NEXT: vpshufb %xmm1, %xmm9, %xmm93091; AVX512-NEXT: vinserti128 $1, %xmm8, %ymm9, %ymm83092; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm10 = zmm8[0,1,2,3],zmm3[0,1,2,3]3093; AVX512-NEXT: vmovdqa (%rdx), %ymm33094; AVX512-NEXT: vmovdqa 32(%rdx), %ymm83095; AVX512-NEXT: vmovdqa 64(%rdx), %ymm143096; AVX512-NEXT: vmovdqa {{.*#+}} ymm9 = [128,128,10,11,128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128]3097; AVX512-NEXT: vpshufb %ymm9, %ymm3, %ymm113098; AVX512-NEXT: vpmovsxbd {{.*#+}} ymm16 = [0,0,0,0,1,1,0,2]3099; AVX512-NEXT: vpermd %ymm3, %ymm16, %ymm33100; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm15 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]3101; AVX512-NEXT: vpandn %ymm3, %ymm15, %ymm33102; AVX512-NEXT: vinserti64x4 $1, %ymm11, %zmm3, %zmm33103; AVX512-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm10 & zmm15)3104; AVX512-NEXT: vmovdqa 96(%rsi), %xmm103105; AVX512-NEXT: vprold $16, %xmm10, %xmm113106; AVX512-NEXT: vmovdqa 96(%rdi), %xmm123107; AVX512-NEXT: vpshufd {{.*#+}} xmm13 = xmm12[1,1,2,2]3108; AVX512-NEXT: vpblendw {{.*#+}} xmm11 = xmm13[0,1],xmm11[2],xmm13[3,4],xmm11[5],xmm13[6,7]3109; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]3110; AVX512-NEXT: vpshufb %xmm1, %xmm10, %xmm103111; AVX512-NEXT: vinserti128 $1, %xmm11, %ymm10, %ymm103112; AVX512-NEXT: vmovdqa 80(%rdi), %xmm123113; AVX512-NEXT: vmovdqa 80(%rsi), %xmm133114; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7]3115; AVX512-NEXT: vmovdqa {{.*#+}} xmm11 = [4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]3116; AVX512-NEXT: vpshufb %xmm11, %xmm0, %xmm03117; AVX512-NEXT: vprold $16, %xmm13, %xmm133118; AVX512-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[1,1,2,2]3119; AVX512-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1],xmm13[2],xmm12[3,4],xmm13[5],xmm12[6,7]3120; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm12, %ymm03121; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm10[0,1,2,3]3122; AVX512-NEXT: vpmovsxbd {{.*#+}} zmm18 = [5,5,0,6,6,0,7,7,0,8,8,0,9,9,0,10]3123; AVX512-NEXT: vpermd 64(%rdx), %zmm18, %zmm103124; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm22 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]3125; AVX512-NEXT: vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm22 & (zmm10 ^ zmm0))3126; AVX512-NEXT: vmovdqa 96(%rdi), %ymm03127; AVX512-NEXT: vmovdqa %ymm6, %ymm23128; AVX512-NEXT: vpshufb %ymm6, %ymm0, %ymm03129; AVX512-NEXT: vmovdqa 96(%rsi), %ymm53130; AVX512-NEXT: vmovdqa %ymm7, %ymm133131; AVX512-NEXT: vpshufb %ymm7, %ymm5, %ymm53132; AVX512-NEXT: vpor %ymm0, %ymm5, %ymm03133; AVX512-NEXT: vmovdqa 112(%rdi), %xmm53134; AVX512-NEXT: vmovdqa 112(%rsi), %xmm123135; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm12[4],xmm5[4],xmm12[5],xmm5[5],xmm12[6],xmm5[6],xmm12[7],xmm5[7]3136; AVX512-NEXT: vpshufb %xmm11, %xmm7, %xmm73137; AVX512-NEXT: vprold $16, %xmm12, %xmm123138; AVX512-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,2]3139; AVX512-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm12[2],xmm5[3,4],xmm12[5],xmm5[6,7]3140; AVX512-NEXT: vinserti128 $1, %xmm7, %ymm5, %ymm53141; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm5[0,1,2,3]3142; AVX512-NEXT: vmovdqa 96(%rdx), %ymm53143; AVX512-NEXT: vpermd %ymm5, %ymm18, %ymm73144; AVX512-NEXT: vpandnq %ymm7, %ymm22, %ymm73145; AVX512-NEXT: vpshufb %ymm9, %ymm5, %ymm53146; AVX512-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm173147; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm19 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]3148; AVX512-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm0 & zmm19)3149; AVX512-NEXT: vmovdqa 64(%rdi), %ymm03150; AVX512-NEXT: vpshufb %ymm6, %ymm0, %ymm03151; AVX512-NEXT: vmovdqa 64(%rsi), %ymm73152; AVX512-NEXT: vpshufb %ymm13, %ymm7, %ymm73153; AVX512-NEXT: vpor %ymm0, %ymm7, %ymm03154; AVX512-NEXT: vmovdqa 64(%rsi), %xmm73155; AVX512-NEXT: vprold $16, %xmm7, %xmm123156; AVX512-NEXT: vmovdqa 64(%rdi), %xmm53157; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[1,1,2,2]3158; AVX512-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm12[2],xmm6[3,4],xmm12[5],xmm6[6,7]3159; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3]3160; AVX512-NEXT: vpshufb %xmm1, %xmm5, %xmm53161; AVX512-NEXT: vinserti128 $1, %xmm6, %ymm5, %ymm53162; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm5[0,1,2,3],zmm0[0,1,2,3]3163; AVX512-NEXT: vpshufb %ymm9, %ymm14, %ymm53164; AVX512-NEXT: vpermd %ymm14, %ymm16, %ymm63165; AVX512-NEXT: vpandn %ymm6, %ymm15, %ymm63166; AVX512-NEXT: vinserti64x4 $1, %ymm5, %zmm6, %zmm53167; AVX512-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm0 & zmm15)3168; AVX512-NEXT: vmovdqa 32(%rdi), %ymm03169; AVX512-NEXT: vpshufb %ymm2, %ymm0, %ymm03170; AVX512-NEXT: vmovdqa 32(%rsi), %ymm63171; AVX512-NEXT: vpshufb %ymm13, %ymm6, %ymm63172; AVX512-NEXT: vpor %ymm0, %ymm6, %ymm03173; AVX512-NEXT: vmovdqa 48(%rdi), %xmm63174; AVX512-NEXT: vmovdqa 48(%rsi), %xmm73175; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]3176; AVX512-NEXT: vpshufb %xmm11, %xmm12, %xmm123177; AVX512-NEXT: vprold $16, %xmm7, %xmm73178; AVX512-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[1,1,2,2]3179; AVX512-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm7[2],xmm6[3,4],xmm7[5],xmm6[6,7]3180; AVX512-NEXT: vinserti128 $1, %xmm12, %ymm6, %ymm63181; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm6[0,1,2,3]3182; AVX512-NEXT: vpermd %ymm8, %ymm18, %ymm63183; AVX512-NEXT: vpandnq %ymm6, %ymm22, %ymm63184; AVX512-NEXT: vpshufb %ymm9, %ymm8, %ymm73185; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm7, %zmm63186; AVX512-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm0 & zmm19)3187; AVX512-NEXT: vmovdqa64 %xmm24, %xmm23188; AVX512-NEXT: vprold $16, %xmm24, %xmm03189; AVX512-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,2,2]3190; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm7[0,1],xmm0[2],xmm7[3,4],xmm0[5],xmm7[6,7]3191; AVX512-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]3192; AVX512-NEXT: vpshufb %xmm1, %xmm2, %xmm23193; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm03194; AVX512-NEXT: vmovdqa64 %xmm20, %xmm43195; AVX512-NEXT: vmovdqa64 %xmm21, %xmm13196; AVX512-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]3197; AVX512-NEXT: vpshufb %xmm11, %xmm2, %xmm23198; AVX512-NEXT: vprold $16, %xmm20, %xmm43199; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm21[1,1,2,2]3200; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm4[2],xmm1[3,4],xmm4[5],xmm1[6,7]3201; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm13202; AVX512-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[0,1,2,3]3203; AVX512-NEXT: vpermd (%rdx), %zmm18, %zmm13204; AVX512-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm22 & (zmm1 ^ zmm0))3205; AVX512-NEXT: vmovdqa64 %zmm1, 64(%rcx)3206; AVX512-NEXT: vmovdqa64 %zmm6, 128(%rcx)3207; AVX512-NEXT: vmovdqa64 %zmm5, 192(%rcx)3208; AVX512-NEXT: vmovdqa64 %zmm17, 320(%rcx)3209; AVX512-NEXT: vmovdqa64 %zmm10, 256(%rcx)3210; AVX512-NEXT: vmovdqa64 %zmm3, (%rcx)3211; AVX512-NEXT: vzeroupper3212; AVX512-NEXT: retq3213;3214; AVX512-FCP-LABEL: store_i16_stride3_vf64:3215; AVX512-FCP: # %bb.0:3216; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm03217; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128,20,21,128,128]3218; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm03219; AVX512-FCP-NEXT: vmovdqa (%rsi), %ymm13220; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [10,11,0,1,128,128,12,13,2,3,128,128,14,15,4,5,128,128,16,17,28,29,128,128,18,19,18,19,128,128,20,21]3221; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm1, %ymm13222; AVX512-FCP-NEXT: vpor %ymm0, %ymm1, %ymm33223; AVX512-FCP-NEXT: vmovdqa (%rsi), %xmm53224; AVX512-FCP-NEXT: vmovdqa64 16(%rsi), %xmm203225; AVX512-FCP-NEXT: vmovdqa64 32(%rsi), %xmm243226; AVX512-FCP-NEXT: vprold $16, %xmm5, %xmm83227; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm93228; AVX512-FCP-NEXT: vmovdqa64 16(%rdi), %xmm213229; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %xmm43230; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm10 = xmm9[1,1,2,2]3231; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm8 = xmm10[0,1],xmm8[2],xmm10[3,4],xmm8[5],xmm10[6,7]3232; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]3233; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]3234; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm9, %xmm93235; AVX512-FCP-NEXT: vinserti128 $1, %xmm8, %ymm9, %ymm83236; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm10 = zmm8[0,1,2,3],zmm3[0,1,2,3]3237; AVX512-FCP-NEXT: vmovdqa (%rdx), %ymm33238; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %ymm83239; AVX512-FCP-NEXT: vmovdqa 64(%rdx), %ymm143240; AVX512-FCP-NEXT: vmovdqa {{.*#+}} ymm9 = [128,128,10,11,128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128]3241; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm3, %ymm113242; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} ymm16 = [0,0,0,0,1,1,0,2]3243; AVX512-FCP-NEXT: vpermd %ymm3, %ymm16, %ymm33244; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm15 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]3245; AVX512-FCP-NEXT: vpandn %ymm3, %ymm15, %ymm33246; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm3, %zmm33247; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm10 & zmm15)3248; AVX512-FCP-NEXT: vmovdqa 96(%rsi), %xmm103249; AVX512-FCP-NEXT: vprold $16, %xmm10, %xmm113250; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %xmm123251; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm13 = xmm12[1,1,2,2]3252; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm11 = xmm13[0,1],xmm11[2],xmm13[3,4],xmm11[5],xmm13[6,7]3253; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]3254; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm103255; AVX512-FCP-NEXT: vinserti128 $1, %xmm11, %ymm10, %ymm103256; AVX512-FCP-NEXT: vmovdqa 80(%rdi), %xmm123257; AVX512-FCP-NEXT: vmovdqa 80(%rsi), %xmm133258; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7]3259; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm11 = [4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]3260; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm0, %xmm03261; AVX512-FCP-NEXT: vprold $16, %xmm13, %xmm133262; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[1,1,2,2]3263; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1],xmm13[2],xmm12[3,4],xmm13[5],xmm12[6,7]3264; AVX512-FCP-NEXT: vinserti128 $1, %xmm0, %ymm12, %ymm03265; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm10[0,1,2,3]3266; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm18 = [5,5,0,6,6,0,7,7,0,8,8,0,9,9,0,10]3267; AVX512-FCP-NEXT: vpermd 64(%rdx), %zmm18, %zmm103268; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm22 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]3269; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm22 & (zmm10 ^ zmm0))3270; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %ymm03271; AVX512-FCP-NEXT: vmovdqa %ymm6, %ymm23272; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm03273; AVX512-FCP-NEXT: vmovdqa 96(%rsi), %ymm53274; AVX512-FCP-NEXT: vmovdqa %ymm7, %ymm133275; AVX512-FCP-NEXT: vpshufb %ymm7, %ymm5, %ymm53276; AVX512-FCP-NEXT: vpor %ymm0, %ymm5, %ymm03277; AVX512-FCP-NEXT: vmovdqa 112(%rdi), %xmm53278; AVX512-FCP-NEXT: vmovdqa 112(%rsi), %xmm123279; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm12[4],xmm5[4],xmm12[5],xmm5[5],xmm12[6],xmm5[6],xmm12[7],xmm5[7]3280; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm7, %xmm73281; AVX512-FCP-NEXT: vprold $16, %xmm12, %xmm123282; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,2]3283; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm12[2],xmm5[3,4],xmm12[5],xmm5[6,7]3284; AVX512-FCP-NEXT: vinserti128 $1, %xmm7, %ymm5, %ymm53285; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm5[0,1,2,3]3286; AVX512-FCP-NEXT: vmovdqa 96(%rdx), %ymm53287; AVX512-FCP-NEXT: vpermd %ymm5, %ymm18, %ymm73288; AVX512-FCP-NEXT: vpandnq %ymm7, %ymm22, %ymm73289; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm5, %ymm53290; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm173291; AVX512-FCP-NEXT: vmovdqa64 {{.*#+}} zmm19 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]3292; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm0 & zmm19)3293; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm03294; AVX512-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm03295; AVX512-FCP-NEXT: vmovdqa 64(%rsi), %ymm73296; AVX512-FCP-NEXT: vpshufb %ymm13, %ymm7, %ymm73297; AVX512-FCP-NEXT: vpor %ymm0, %ymm7, %ymm03298; AVX512-FCP-NEXT: vmovdqa 64(%rsi), %xmm73299; AVX512-FCP-NEXT: vprold $16, %xmm7, %xmm123300; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %xmm53301; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[1,1,2,2]3302; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm12[2],xmm6[3,4],xmm12[5],xmm6[6,7]3303; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3]3304; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm5, %xmm53305; AVX512-FCP-NEXT: vinserti128 $1, %xmm6, %ymm5, %ymm53306; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm5[0,1,2,3],zmm0[0,1,2,3]3307; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm14, %ymm53308; AVX512-FCP-NEXT: vpermd %ymm14, %ymm16, %ymm63309; AVX512-FCP-NEXT: vpandn %ymm6, %ymm15, %ymm63310; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm5, %zmm6, %zmm53311; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm0 & zmm15)3312; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm03313; AVX512-FCP-NEXT: vpshufb %ymm2, %ymm0, %ymm03314; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %ymm63315; AVX512-FCP-NEXT: vpshufb %ymm13, %ymm6, %ymm63316; AVX512-FCP-NEXT: vpor %ymm0, %ymm6, %ymm03317; AVX512-FCP-NEXT: vmovdqa 48(%rdi), %xmm63318; AVX512-FCP-NEXT: vmovdqa 48(%rsi), %xmm73319; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]3320; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm12, %xmm123321; AVX512-FCP-NEXT: vprold $16, %xmm7, %xmm73322; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[1,1,2,2]3323; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm7[2],xmm6[3,4],xmm7[5],xmm6[6,7]3324; AVX512-FCP-NEXT: vinserti128 $1, %xmm12, %ymm6, %ymm63325; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm6[0,1,2,3]3326; AVX512-FCP-NEXT: vpermd %ymm8, %ymm18, %ymm63327; AVX512-FCP-NEXT: vpandnq %ymm6, %ymm22, %ymm63328; AVX512-FCP-NEXT: vpshufb %ymm9, %ymm8, %ymm73329; AVX512-FCP-NEXT: vinserti64x4 $1, %ymm6, %zmm7, %zmm63330; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm0 & zmm19)3331; AVX512-FCP-NEXT: vmovdqa64 %xmm24, %xmm23332; AVX512-FCP-NEXT: vprold $16, %xmm24, %xmm03333; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,2,2]3334; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm7[0,1],xmm0[2],xmm7[3,4],xmm0[5],xmm7[6,7]3335; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]3336; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm23337; AVX512-FCP-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm03338; AVX512-FCP-NEXT: vmovdqa64 %xmm20, %xmm43339; AVX512-FCP-NEXT: vmovdqa64 %xmm21, %xmm13340; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]3341; AVX512-FCP-NEXT: vpshufb %xmm11, %xmm2, %xmm23342; AVX512-FCP-NEXT: vprold $16, %xmm20, %xmm43343; AVX512-FCP-NEXT: vpshufd {{.*#+}} xmm1 = xmm21[1,1,2,2]3344; AVX512-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm4[2],xmm1[3,4],xmm4[5],xmm1[6,7]3345; AVX512-FCP-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm13346; AVX512-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[0,1,2,3]3347; AVX512-FCP-NEXT: vpermd (%rdx), %zmm18, %zmm13348; AVX512-FCP-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm22 & (zmm1 ^ zmm0))3349; AVX512-FCP-NEXT: vmovdqa64 %zmm1, 64(%rcx)3350; AVX512-FCP-NEXT: vmovdqa64 %zmm6, 128(%rcx)3351; AVX512-FCP-NEXT: vmovdqa64 %zmm5, 192(%rcx)3352; AVX512-FCP-NEXT: vmovdqa64 %zmm17, 320(%rcx)3353; AVX512-FCP-NEXT: vmovdqa64 %zmm10, 256(%rcx)3354; AVX512-FCP-NEXT: vmovdqa64 %zmm3, (%rcx)3355; AVX512-FCP-NEXT: vzeroupper3356; AVX512-FCP-NEXT: retq3357;3358; AVX512DQ-LABEL: store_i16_stride3_vf64:3359; AVX512DQ: # %bb.0:3360; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm03361; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm6 = [128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128,20,21,128,128]3362; AVX512DQ-NEXT: vpshufb %ymm6, %ymm0, %ymm03363; AVX512DQ-NEXT: vmovdqa (%rsi), %ymm13364; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm7 = [10,11,0,1,128,128,12,13,2,3,128,128,14,15,4,5,128,128,16,17,28,29,128,128,18,19,18,19,128,128,20,21]3365; AVX512DQ-NEXT: vpshufb %ymm7, %ymm1, %ymm13366; AVX512DQ-NEXT: vpor %ymm0, %ymm1, %ymm33367; AVX512DQ-NEXT: vmovdqa (%rsi), %xmm53368; AVX512DQ-NEXT: vmovdqa64 16(%rsi), %xmm203369; AVX512DQ-NEXT: vmovdqa64 32(%rsi), %xmm243370; AVX512DQ-NEXT: vprold $16, %xmm5, %xmm83371; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm93372; AVX512DQ-NEXT: vmovdqa64 16(%rdi), %xmm213373; AVX512DQ-NEXT: vmovdqa 32(%rdi), %xmm43374; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm10 = xmm9[1,1,2,2]3375; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm8 = xmm10[0,1],xmm8[2],xmm10[3,4],xmm8[5],xmm10[6,7]3376; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]3377; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]3378; AVX512DQ-NEXT: vpshufb %xmm1, %xmm9, %xmm93379; AVX512DQ-NEXT: vinserti128 $1, %xmm8, %ymm9, %ymm83380; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm10 = zmm8[0,1,2,3],zmm3[0,1,2,3]3381; AVX512DQ-NEXT: vmovdqa (%rdx), %ymm33382; AVX512DQ-NEXT: vmovdqa 32(%rdx), %ymm83383; AVX512DQ-NEXT: vmovdqa 64(%rdx), %ymm143384; AVX512DQ-NEXT: vmovdqa {{.*#+}} ymm9 = [128,128,10,11,128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128]3385; AVX512DQ-NEXT: vpshufb %ymm9, %ymm3, %ymm113386; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} ymm16 = [0,0,0,0,1,1,0,2]3387; AVX512DQ-NEXT: vpermd %ymm3, %ymm16, %ymm33388; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm15 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]3389; AVX512DQ-NEXT: vpandn %ymm3, %ymm15, %ymm33390; AVX512DQ-NEXT: vinserti64x4 $1, %ymm11, %zmm3, %zmm33391; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm10 & zmm15)3392; AVX512DQ-NEXT: vmovdqa 96(%rsi), %xmm103393; AVX512DQ-NEXT: vprold $16, %xmm10, %xmm113394; AVX512DQ-NEXT: vmovdqa 96(%rdi), %xmm123395; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm13 = xmm12[1,1,2,2]3396; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm11 = xmm13[0,1],xmm11[2],xmm13[3,4],xmm11[5],xmm13[6,7]3397; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]3398; AVX512DQ-NEXT: vpshufb %xmm1, %xmm10, %xmm103399; AVX512DQ-NEXT: vinserti128 $1, %xmm11, %ymm10, %ymm103400; AVX512DQ-NEXT: vmovdqa 80(%rdi), %xmm123401; AVX512DQ-NEXT: vmovdqa 80(%rsi), %xmm133402; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7]3403; AVX512DQ-NEXT: vmovdqa {{.*#+}} xmm11 = [4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]3404; AVX512DQ-NEXT: vpshufb %xmm11, %xmm0, %xmm03405; AVX512DQ-NEXT: vprold $16, %xmm13, %xmm133406; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[1,1,2,2]3407; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1],xmm13[2],xmm12[3,4],xmm13[5],xmm12[6,7]3408; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm12, %ymm03409; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm10[0,1,2,3]3410; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm18 = [5,5,0,6,6,0,7,7,0,8,8,0,9,9,0,10]3411; AVX512DQ-NEXT: vpermd 64(%rdx), %zmm18, %zmm103412; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm22 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]3413; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm22 & (zmm10 ^ zmm0))3414; AVX512DQ-NEXT: vmovdqa 96(%rdi), %ymm03415; AVX512DQ-NEXT: vmovdqa %ymm6, %ymm23416; AVX512DQ-NEXT: vpshufb %ymm6, %ymm0, %ymm03417; AVX512DQ-NEXT: vmovdqa 96(%rsi), %ymm53418; AVX512DQ-NEXT: vmovdqa %ymm7, %ymm133419; AVX512DQ-NEXT: vpshufb %ymm7, %ymm5, %ymm53420; AVX512DQ-NEXT: vpor %ymm0, %ymm5, %ymm03421; AVX512DQ-NEXT: vmovdqa 112(%rdi), %xmm53422; AVX512DQ-NEXT: vmovdqa 112(%rsi), %xmm123423; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm12[4],xmm5[4],xmm12[5],xmm5[5],xmm12[6],xmm5[6],xmm12[7],xmm5[7]3424; AVX512DQ-NEXT: vpshufb %xmm11, %xmm7, %xmm73425; AVX512DQ-NEXT: vprold $16, %xmm12, %xmm123426; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,2]3427; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm12[2],xmm5[3,4],xmm12[5],xmm5[6,7]3428; AVX512DQ-NEXT: vinserti128 $1, %xmm7, %ymm5, %ymm53429; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm5[0,1,2,3]3430; AVX512DQ-NEXT: vmovdqa 96(%rdx), %ymm53431; AVX512DQ-NEXT: vpermd %ymm5, %ymm18, %ymm73432; AVX512DQ-NEXT: vpandnq %ymm7, %ymm22, %ymm73433; AVX512DQ-NEXT: vpshufb %ymm9, %ymm5, %ymm53434; AVX512DQ-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm173435; AVX512DQ-NEXT: vmovdqa64 {{.*#+}} zmm19 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]3436; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm0 & zmm19)3437; AVX512DQ-NEXT: vmovdqa 64(%rdi), %ymm03438; AVX512DQ-NEXT: vpshufb %ymm6, %ymm0, %ymm03439; AVX512DQ-NEXT: vmovdqa 64(%rsi), %ymm73440; AVX512DQ-NEXT: vpshufb %ymm13, %ymm7, %ymm73441; AVX512DQ-NEXT: vpor %ymm0, %ymm7, %ymm03442; AVX512DQ-NEXT: vmovdqa 64(%rsi), %xmm73443; AVX512DQ-NEXT: vprold $16, %xmm7, %xmm123444; AVX512DQ-NEXT: vmovdqa 64(%rdi), %xmm53445; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[1,1,2,2]3446; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm12[2],xmm6[3,4],xmm12[5],xmm6[6,7]3447; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3]3448; AVX512DQ-NEXT: vpshufb %xmm1, %xmm5, %xmm53449; AVX512DQ-NEXT: vinserti128 $1, %xmm6, %ymm5, %ymm53450; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm5[0,1,2,3],zmm0[0,1,2,3]3451; AVX512DQ-NEXT: vpshufb %ymm9, %ymm14, %ymm53452; AVX512DQ-NEXT: vpermd %ymm14, %ymm16, %ymm63453; AVX512DQ-NEXT: vpandn %ymm6, %ymm15, %ymm63454; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm6, %zmm53455; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm0 & zmm15)3456; AVX512DQ-NEXT: vmovdqa 32(%rdi), %ymm03457; AVX512DQ-NEXT: vpshufb %ymm2, %ymm0, %ymm03458; AVX512DQ-NEXT: vmovdqa 32(%rsi), %ymm63459; AVX512DQ-NEXT: vpshufb %ymm13, %ymm6, %ymm63460; AVX512DQ-NEXT: vpor %ymm0, %ymm6, %ymm03461; AVX512DQ-NEXT: vmovdqa 48(%rdi), %xmm63462; AVX512DQ-NEXT: vmovdqa 48(%rsi), %xmm73463; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]3464; AVX512DQ-NEXT: vpshufb %xmm11, %xmm12, %xmm123465; AVX512DQ-NEXT: vprold $16, %xmm7, %xmm73466; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[1,1,2,2]3467; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm7[2],xmm6[3,4],xmm7[5],xmm6[6,7]3468; AVX512DQ-NEXT: vinserti128 $1, %xmm12, %ymm6, %ymm63469; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm6[0,1,2,3]3470; AVX512DQ-NEXT: vpermd %ymm8, %ymm18, %ymm63471; AVX512DQ-NEXT: vpandnq %ymm6, %ymm22, %ymm63472; AVX512DQ-NEXT: vpshufb %ymm9, %ymm8, %ymm73473; AVX512DQ-NEXT: vinserti64x4 $1, %ymm6, %zmm7, %zmm63474; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm0 & zmm19)3475; AVX512DQ-NEXT: vmovdqa64 %xmm24, %xmm23476; AVX512DQ-NEXT: vprold $16, %xmm24, %xmm03477; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,2,2]3478; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm7[0,1],xmm0[2],xmm7[3,4],xmm0[5],xmm7[6,7]3479; AVX512DQ-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]3480; AVX512DQ-NEXT: vpshufb %xmm1, %xmm2, %xmm23481; AVX512DQ-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm03482; AVX512DQ-NEXT: vmovdqa64 %xmm20, %xmm43483; AVX512DQ-NEXT: vmovdqa64 %xmm21, %xmm13484; AVX512DQ-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]3485; AVX512DQ-NEXT: vpshufb %xmm11, %xmm2, %xmm23486; AVX512DQ-NEXT: vprold $16, %xmm20, %xmm43487; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm21[1,1,2,2]3488; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm4[2],xmm1[3,4],xmm4[5],xmm1[6,7]3489; AVX512DQ-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm13490; AVX512DQ-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[0,1,2,3]3491; AVX512DQ-NEXT: vpermd (%rdx), %zmm18, %zmm13492; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm22 & (zmm1 ^ zmm0))3493; AVX512DQ-NEXT: vmovdqa64 %zmm1, 64(%rcx)3494; AVX512DQ-NEXT: vmovdqa64 %zmm6, 128(%rcx)3495; AVX512DQ-NEXT: vmovdqa64 %zmm5, 192(%rcx)3496; AVX512DQ-NEXT: vmovdqa64 %zmm17, 320(%rcx)3497; AVX512DQ-NEXT: vmovdqa64 %zmm10, 256(%rcx)3498; AVX512DQ-NEXT: vmovdqa64 %zmm3, (%rcx)3499; AVX512DQ-NEXT: vzeroupper3500; AVX512DQ-NEXT: retq3501;3502; AVX512DQ-FCP-LABEL: store_i16_stride3_vf64:3503; AVX512DQ-FCP: # %bb.0:3504; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm03505; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm6 = [128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128,20,21,128,128]3506; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm03507; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %ymm13508; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm7 = [10,11,0,1,128,128,12,13,2,3,128,128,14,15,4,5,128,128,16,17,28,29,128,128,18,19,18,19,128,128,20,21]3509; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm1, %ymm13510; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm1, %ymm33511; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %xmm53512; AVX512DQ-FCP-NEXT: vmovdqa64 16(%rsi), %xmm203513; AVX512DQ-FCP-NEXT: vmovdqa64 32(%rsi), %xmm243514; AVX512DQ-FCP-NEXT: vprold $16, %xmm5, %xmm83515; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm93516; AVX512DQ-FCP-NEXT: vmovdqa64 16(%rdi), %xmm213517; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %xmm43518; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm10 = xmm9[1,1,2,2]3519; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm8 = xmm10[0,1],xmm8[2],xmm10[3,4],xmm8[5],xmm10[6,7]3520; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]3521; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]3522; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm9, %xmm93523; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm8, %ymm9, %ymm83524; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm10 = zmm8[0,1,2,3],zmm3[0,1,2,3]3525; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %ymm33526; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %ymm83527; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdx), %ymm143528; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} ymm9 = [128,128,10,11,128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128]3529; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm3, %ymm113530; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} ymm16 = [0,0,0,0,1,1,0,2]3531; AVX512DQ-FCP-NEXT: vpermd %ymm3, %ymm16, %ymm33532; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm15 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]3533; AVX512DQ-FCP-NEXT: vpandn %ymm3, %ymm15, %ymm33534; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm11, %zmm3, %zmm33535; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm3 = zmm3 | (zmm10 & zmm15)3536; AVX512DQ-FCP-NEXT: vmovdqa 96(%rsi), %xmm103537; AVX512DQ-FCP-NEXT: vprold $16, %xmm10, %xmm113538; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %xmm123539; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm13 = xmm12[1,1,2,2]3540; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm11 = xmm13[0,1],xmm11[2],xmm13[3,4],xmm11[5],xmm13[6,7]3541; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]3542; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm10, %xmm103543; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm11, %ymm10, %ymm103544; AVX512DQ-FCP-NEXT: vmovdqa 80(%rdi), %xmm123545; AVX512DQ-FCP-NEXT: vmovdqa 80(%rsi), %xmm133546; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7]3547; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm11 = [4,5,10,11,10,11,8,9,8,9,14,15,12,13,14,15]3548; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm0, %xmm03549; AVX512DQ-FCP-NEXT: vprold $16, %xmm13, %xmm133550; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm12 = xmm12[1,1,2,2]3551; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm12 = xmm12[0,1],xmm13[2],xmm12[3,4],xmm13[5],xmm12[6,7]3552; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm0, %ymm12, %ymm03553; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm10[0,1,2,3]3554; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm18 = [5,5,0,6,6,0,7,7,0,8,8,0,9,9,0,10]3555; AVX512DQ-FCP-NEXT: vpermd 64(%rdx), %zmm18, %zmm103556; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm22 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]3557; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm10 = zmm10 ^ (zmm22 & (zmm10 ^ zmm0))3558; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %ymm03559; AVX512DQ-FCP-NEXT: vmovdqa %ymm6, %ymm23560; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm03561; AVX512DQ-FCP-NEXT: vmovdqa 96(%rsi), %ymm53562; AVX512DQ-FCP-NEXT: vmovdqa %ymm7, %ymm133563; AVX512DQ-FCP-NEXT: vpshufb %ymm7, %ymm5, %ymm53564; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm5, %ymm03565; AVX512DQ-FCP-NEXT: vmovdqa 112(%rdi), %xmm53566; AVX512DQ-FCP-NEXT: vmovdqa 112(%rsi), %xmm123567; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm12[4],xmm5[4],xmm12[5],xmm5[5],xmm12[6],xmm5[6],xmm12[7],xmm5[7]3568; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm7, %xmm73569; AVX512DQ-FCP-NEXT: vprold $16, %xmm12, %xmm123570; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,2]3571; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm12[2],xmm5[3,4],xmm12[5],xmm5[6,7]3572; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm7, %ymm5, %ymm53573; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm5[0,1,2,3]3574; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdx), %ymm53575; AVX512DQ-FCP-NEXT: vpermd %ymm5, %ymm18, %ymm73576; AVX512DQ-FCP-NEXT: vpandnq %ymm7, %ymm22, %ymm73577; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm5, %ymm53578; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm7, %zmm5, %zmm173579; AVX512DQ-FCP-NEXT: vmovdqa64 {{.*#+}} zmm19 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0]3580; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm17 = zmm17 | (zmm0 & zmm19)3581; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm03582; AVX512DQ-FCP-NEXT: vpshufb %ymm6, %ymm0, %ymm03583; AVX512DQ-FCP-NEXT: vmovdqa 64(%rsi), %ymm73584; AVX512DQ-FCP-NEXT: vpshufb %ymm13, %ymm7, %ymm73585; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm7, %ymm03586; AVX512DQ-FCP-NEXT: vmovdqa 64(%rsi), %xmm73587; AVX512DQ-FCP-NEXT: vprold $16, %xmm7, %xmm123588; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %xmm53589; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[1,1,2,2]3590; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm12[2],xmm6[3,4],xmm12[5],xmm6[6,7]3591; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3]3592; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm5, %xmm53593; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm6, %ymm5, %ymm53594; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm5[0,1,2,3],zmm0[0,1,2,3]3595; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm14, %ymm53596; AVX512DQ-FCP-NEXT: vpermd %ymm14, %ymm16, %ymm63597; AVX512DQ-FCP-NEXT: vpandn %ymm6, %ymm15, %ymm63598; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm5, %zmm6, %zmm53599; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm5 = zmm5 | (zmm0 & zmm15)3600; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm03601; AVX512DQ-FCP-NEXT: vpshufb %ymm2, %ymm0, %ymm03602; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %ymm63603; AVX512DQ-FCP-NEXT: vpshufb %ymm13, %ymm6, %ymm63604; AVX512DQ-FCP-NEXT: vpor %ymm0, %ymm6, %ymm03605; AVX512DQ-FCP-NEXT: vmovdqa 48(%rdi), %xmm63606; AVX512DQ-FCP-NEXT: vmovdqa 48(%rsi), %xmm73607; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm12 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]3608; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm12, %xmm123609; AVX512DQ-FCP-NEXT: vprold $16, %xmm7, %xmm73610; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[1,1,2,2]3611; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm7[2],xmm6[3,4],xmm7[5],xmm6[6,7]3612; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm12, %ymm6, %ymm63613; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm6[0,1,2,3]3614; AVX512DQ-FCP-NEXT: vpermd %ymm8, %ymm18, %ymm63615; AVX512DQ-FCP-NEXT: vpandnq %ymm6, %ymm22, %ymm63616; AVX512DQ-FCP-NEXT: vpshufb %ymm9, %ymm8, %ymm73617; AVX512DQ-FCP-NEXT: vinserti64x4 $1, %ymm6, %zmm7, %zmm63618; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm6 = zmm6 | (zmm0 & zmm19)3619; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm24, %xmm23620; AVX512DQ-FCP-NEXT: vprold $16, %xmm24, %xmm03621; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm7 = xmm4[1,1,2,2]3622; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm0 = xmm7[0,1],xmm0[2],xmm7[3,4],xmm0[5],xmm7[6,7]3623; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]3624; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm2, %xmm23625; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm03626; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm20, %xmm43627; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm21, %xmm13628; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]3629; AVX512DQ-FCP-NEXT: vpshufb %xmm11, %xmm2, %xmm23630; AVX512DQ-FCP-NEXT: vprold $16, %xmm20, %xmm43631; AVX512DQ-FCP-NEXT: vpshufd {{.*#+}} xmm1 = xmm21[1,1,2,2]3632; AVX512DQ-FCP-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm4[2],xmm1[3,4],xmm4[5],xmm1[6,7]3633; AVX512DQ-FCP-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm13634; AVX512DQ-FCP-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[0,1,2,3]3635; AVX512DQ-FCP-NEXT: vpermd (%rdx), %zmm18, %zmm13636; AVX512DQ-FCP-NEXT: vpternlogq {{.*#+}} zmm1 = zmm1 ^ (zmm22 & (zmm1 ^ zmm0))3637; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, 64(%rcx)3638; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm6, 128(%rcx)3639; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, 192(%rcx)3640; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm17, 320(%rcx)3641; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm10, 256(%rcx)3642; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, (%rcx)3643; AVX512DQ-FCP-NEXT: vzeroupper3644; AVX512DQ-FCP-NEXT: retq3645;3646; AVX512BW-LABEL: store_i16_stride3_vf64:3647; AVX512BW: # %bb.0:3648; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm03649; AVX512BW-NEXT: vmovdqa64 64(%rdi), %zmm13650; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm23651; AVX512BW-NEXT: vmovdqa64 64(%rsi), %zmm33652; AVX512BW-NEXT: vmovdqa64 (%rdx), %zmm43653; AVX512BW-NEXT: vmovdqa64 64(%rdx), %zmm53654; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm6 = [0,32,0,1,33,0,2,34,0,3,35,0,4,36,0,5,37,0,6,38,0,7,39,0,8,40,0,9,41,0,10,42]3655; AVX512BW-NEXT: vmovdqa64 %zmm0, %zmm73656; AVX512BW-NEXT: vpermt2w %zmm2, %zmm6, %zmm73657; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,32,3,4,33,6,7,34,9,10,35,12,13,36,15,16,37,18,19,38,21,22,39,24,25,40,27,28,41,30,31]3658; AVX512BW-NEXT: vpermt2w %zmm4, %zmm8, %zmm73659; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm9 = [21,0,54,22,0,55,23,0,56,24,0,57,25,0,58,26,0,59,27,0,60,28,0,61,29,0,62,30,0,63,31,0]3660; AVX512BW-NEXT: vmovdqa64 %zmm3, %zmm103661; AVX512BW-NEXT: vpermt2w %zmm1, %zmm9, %zmm103662; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm11 = [0,53,2,3,54,5,6,55,8,9,56,11,12,57,14,15,58,17,18,59,20,21,60,23,24,61,26,27,62,29,30,63]3663; AVX512BW-NEXT: vpermt2w %zmm5, %zmm11, %zmm103664; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm12 = [0,11,43,0,12,44,0,13,45,0,14,46,0,15,47,0,16,48,0,17,49,0,18,50,0,19,51,0,20,52,0,21]3665; AVX512BW-NEXT: vmovdqa64 %zmm1, %zmm133666; AVX512BW-NEXT: vpermt2w %zmm3, %zmm12, %zmm133667; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm14 = [42,1,2,43,4,5,44,7,8,45,10,11,46,13,14,47,16,17,48,19,20,49,22,23,50,25,26,51,28,29,52,31]3668; AVX512BW-NEXT: vpermt2w %zmm5, %zmm14, %zmm133669; AVX512BW-NEXT: vpermt2w %zmm3, %zmm6, %zmm13670; AVX512BW-NEXT: vpermt2w %zmm5, %zmm8, %zmm13671; AVX512BW-NEXT: vpermi2w %zmm0, %zmm2, %zmm93672; AVX512BW-NEXT: vpermt2w %zmm4, %zmm11, %zmm93673; AVX512BW-NEXT: vpermt2w %zmm2, %zmm12, %zmm03674; AVX512BW-NEXT: vpermt2w %zmm4, %zmm14, %zmm03675; AVX512BW-NEXT: vmovdqa64 %zmm0, 64(%rcx)3676; AVX512BW-NEXT: vmovdqa64 %zmm9, 128(%rcx)3677; AVX512BW-NEXT: vmovdqa64 %zmm1, 192(%rcx)3678; AVX512BW-NEXT: vmovdqa64 %zmm13, 256(%rcx)3679; AVX512BW-NEXT: vmovdqa64 %zmm10, 320(%rcx)3680; AVX512BW-NEXT: vmovdqa64 %zmm7, (%rcx)3681; AVX512BW-NEXT: vzeroupper3682; AVX512BW-NEXT: retq3683;3684; AVX512BW-FCP-LABEL: store_i16_stride3_vf64:3685; AVX512BW-FCP: # %bb.0:3686; AVX512BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm03687; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm13688; AVX512BW-FCP-NEXT: vmovdqa64 (%rsi), %zmm23689; AVX512BW-FCP-NEXT: vmovdqa64 64(%rsi), %zmm33690; AVX512BW-FCP-NEXT: vmovdqa64 (%rdx), %zmm43691; AVX512BW-FCP-NEXT: vmovdqa64 64(%rdx), %zmm53692; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm6 = [0,32,0,1,33,0,2,34,0,3,35,0,4,36,0,5,37,0,6,38,0,7,39,0,8,40,0,9,41,0,10,42]3693; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm73694; AVX512BW-FCP-NEXT: vpermt2w %zmm2, %zmm6, %zmm73695; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,32,3,4,33,6,7,34,9,10,35,12,13,36,15,16,37,18,19,38,21,22,39,24,25,40,27,28,41,30,31]3696; AVX512BW-FCP-NEXT: vpermt2w %zmm4, %zmm8, %zmm73697; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm9 = [21,0,54,22,0,55,23,0,56,24,0,57,25,0,58,26,0,59,27,0,60,28,0,61,29,0,62,30,0,63,31,0]3698; AVX512BW-FCP-NEXT: vmovdqa64 %zmm3, %zmm103699; AVX512BW-FCP-NEXT: vpermt2w %zmm1, %zmm9, %zmm103700; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm11 = [0,53,2,3,54,5,6,55,8,9,56,11,12,57,14,15,58,17,18,59,20,21,60,23,24,61,26,27,62,29,30,63]3701; AVX512BW-FCP-NEXT: vpermt2w %zmm5, %zmm11, %zmm103702; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm12 = [0,11,43,0,12,44,0,13,45,0,14,46,0,15,47,0,16,48,0,17,49,0,18,50,0,19,51,0,20,52,0,21]3703; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm133704; AVX512BW-FCP-NEXT: vpermt2w %zmm3, %zmm12, %zmm133705; AVX512BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm14 = [42,1,2,43,4,5,44,7,8,45,10,11,46,13,14,47,16,17,48,19,20,49,22,23,50,25,26,51,28,29,52,31]3706; AVX512BW-FCP-NEXT: vpermt2w %zmm5, %zmm14, %zmm133707; AVX512BW-FCP-NEXT: vpermt2w %zmm3, %zmm6, %zmm13708; AVX512BW-FCP-NEXT: vpermt2w %zmm5, %zmm8, %zmm13709; AVX512BW-FCP-NEXT: vpermi2w %zmm0, %zmm2, %zmm93710; AVX512BW-FCP-NEXT: vpermt2w %zmm4, %zmm11, %zmm93711; AVX512BW-FCP-NEXT: vpermt2w %zmm2, %zmm12, %zmm03712; AVX512BW-FCP-NEXT: vpermt2w %zmm4, %zmm14, %zmm03713; AVX512BW-FCP-NEXT: vmovdqa64 %zmm0, 64(%rcx)3714; AVX512BW-FCP-NEXT: vmovdqa64 %zmm9, 128(%rcx)3715; AVX512BW-FCP-NEXT: vmovdqa64 %zmm1, 192(%rcx)3716; AVX512BW-FCP-NEXT: vmovdqa64 %zmm13, 256(%rcx)3717; AVX512BW-FCP-NEXT: vmovdqa64 %zmm10, 320(%rcx)3718; AVX512BW-FCP-NEXT: vmovdqa64 %zmm7, (%rcx)3719; AVX512BW-FCP-NEXT: vzeroupper3720; AVX512BW-FCP-NEXT: retq3721;3722; AVX512DQ-BW-LABEL: store_i16_stride3_vf64:3723; AVX512DQ-BW: # %bb.0:3724; AVX512DQ-BW-NEXT: vmovdqa64 (%rdi), %zmm03725; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdi), %zmm13726; AVX512DQ-BW-NEXT: vmovdqa64 (%rsi), %zmm23727; AVX512DQ-BW-NEXT: vmovdqa64 64(%rsi), %zmm33728; AVX512DQ-BW-NEXT: vmovdqa64 (%rdx), %zmm43729; AVX512DQ-BW-NEXT: vmovdqa64 64(%rdx), %zmm53730; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm6 = [0,32,0,1,33,0,2,34,0,3,35,0,4,36,0,5,37,0,6,38,0,7,39,0,8,40,0,9,41,0,10,42]3731; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, %zmm73732; AVX512DQ-BW-NEXT: vpermt2w %zmm2, %zmm6, %zmm73733; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,32,3,4,33,6,7,34,9,10,35,12,13,36,15,16,37,18,19,38,21,22,39,24,25,40,27,28,41,30,31]3734; AVX512DQ-BW-NEXT: vpermt2w %zmm4, %zmm8, %zmm73735; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm9 = [21,0,54,22,0,55,23,0,56,24,0,57,25,0,58,26,0,59,27,0,60,28,0,61,29,0,62,30,0,63,31,0]3736; AVX512DQ-BW-NEXT: vmovdqa64 %zmm3, %zmm103737; AVX512DQ-BW-NEXT: vpermt2w %zmm1, %zmm9, %zmm103738; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm11 = [0,53,2,3,54,5,6,55,8,9,56,11,12,57,14,15,58,17,18,59,20,21,60,23,24,61,26,27,62,29,30,63]3739; AVX512DQ-BW-NEXT: vpermt2w %zmm5, %zmm11, %zmm103740; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm12 = [0,11,43,0,12,44,0,13,45,0,14,46,0,15,47,0,16,48,0,17,49,0,18,50,0,19,51,0,20,52,0,21]3741; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, %zmm133742; AVX512DQ-BW-NEXT: vpermt2w %zmm3, %zmm12, %zmm133743; AVX512DQ-BW-NEXT: vpmovsxbw {{.*#+}} zmm14 = [42,1,2,43,4,5,44,7,8,45,10,11,46,13,14,47,16,17,48,19,20,49,22,23,50,25,26,51,28,29,52,31]3744; AVX512DQ-BW-NEXT: vpermt2w %zmm5, %zmm14, %zmm133745; AVX512DQ-BW-NEXT: vpermt2w %zmm3, %zmm6, %zmm13746; AVX512DQ-BW-NEXT: vpermt2w %zmm5, %zmm8, %zmm13747; AVX512DQ-BW-NEXT: vpermi2w %zmm0, %zmm2, %zmm93748; AVX512DQ-BW-NEXT: vpermt2w %zmm4, %zmm11, %zmm93749; AVX512DQ-BW-NEXT: vpermt2w %zmm2, %zmm12, %zmm03750; AVX512DQ-BW-NEXT: vpermt2w %zmm4, %zmm14, %zmm03751; AVX512DQ-BW-NEXT: vmovdqa64 %zmm0, 64(%rcx)3752; AVX512DQ-BW-NEXT: vmovdqa64 %zmm9, 128(%rcx)3753; AVX512DQ-BW-NEXT: vmovdqa64 %zmm1, 192(%rcx)3754; AVX512DQ-BW-NEXT: vmovdqa64 %zmm13, 256(%rcx)3755; AVX512DQ-BW-NEXT: vmovdqa64 %zmm10, 320(%rcx)3756; AVX512DQ-BW-NEXT: vmovdqa64 %zmm7, (%rcx)3757; AVX512DQ-BW-NEXT: vzeroupper3758; AVX512DQ-BW-NEXT: retq3759;3760; AVX512DQ-BW-FCP-LABEL: store_i16_stride3_vf64:3761; AVX512DQ-BW-FCP: # %bb.0:3762; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdi), %zmm03763; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdi), %zmm13764; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rsi), %zmm23765; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rsi), %zmm33766; AVX512DQ-BW-FCP-NEXT: vmovdqa64 (%rdx), %zmm43767; AVX512DQ-BW-FCP-NEXT: vmovdqa64 64(%rdx), %zmm53768; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm6 = [0,32,0,1,33,0,2,34,0,3,35,0,4,36,0,5,37,0,6,38,0,7,39,0,8,40,0,9,41,0,10,42]3769; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, %zmm73770; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm2, %zmm6, %zmm73771; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm8 = [0,1,32,3,4,33,6,7,34,9,10,35,12,13,36,15,16,37,18,19,38,21,22,39,24,25,40,27,28,41,30,31]3772; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm4, %zmm8, %zmm73773; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm9 = [21,0,54,22,0,55,23,0,56,24,0,57,25,0,58,26,0,59,27,0,60,28,0,61,29,0,62,30,0,63,31,0]3774; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm3, %zmm103775; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm1, %zmm9, %zmm103776; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm11 = [0,53,2,3,54,5,6,55,8,9,56,11,12,57,14,15,58,17,18,59,20,21,60,23,24,61,26,27,62,29,30,63]3777; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm5, %zmm11, %zmm103778; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm12 = [0,11,43,0,12,44,0,13,45,0,14,46,0,15,47,0,16,48,0,17,49,0,18,50,0,19,51,0,20,52,0,21]3779; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, %zmm133780; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm3, %zmm12, %zmm133781; AVX512DQ-BW-FCP-NEXT: vpmovsxbw {{.*#+}} zmm14 = [42,1,2,43,4,5,44,7,8,45,10,11,46,13,14,47,16,17,48,19,20,49,22,23,50,25,26,51,28,29,52,31]3782; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm5, %zmm14, %zmm133783; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm3, %zmm6, %zmm13784; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm5, %zmm8, %zmm13785; AVX512DQ-BW-FCP-NEXT: vpermi2w %zmm0, %zmm2, %zmm93786; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm4, %zmm11, %zmm93787; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm2, %zmm12, %zmm03788; AVX512DQ-BW-FCP-NEXT: vpermt2w %zmm4, %zmm14, %zmm03789; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm0, 64(%rcx)3790; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm9, 128(%rcx)3791; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm1, 192(%rcx)3792; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm13, 256(%rcx)3793; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm10, 320(%rcx)3794; AVX512DQ-BW-FCP-NEXT: vmovdqa64 %zmm7, (%rcx)3795; AVX512DQ-BW-FCP-NEXT: vzeroupper3796; AVX512DQ-BW-FCP-NEXT: retq3797 %in.vec0 = load <64 x i16>, ptr %in.vecptr0, align 643798 %in.vec1 = load <64 x i16>, ptr %in.vecptr1, align 643799 %in.vec2 = load <64 x i16>, ptr %in.vecptr2, align 643800 %1 = shufflevector <64 x i16> %in.vec0, <64 x i16> %in.vec1, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>3801 %2 = shufflevector <64 x i16> %in.vec2, <64 x i16> poison, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>3802 %3 = shufflevector <128 x i16> %1, <128 x i16> %2, <192 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127, i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143, i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151, i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159, i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191>3803 %interleaved.vec = shufflevector <192 x i16> %3, <192 x i16> poison, <192 x i32> <i32 0, i32 64, i32 128, i32 1, i32 65, i32 129, i32 2, i32 66, i32 130, i32 3, i32 67, i32 131, i32 4, i32 68, i32 132, i32 5, i32 69, i32 133, i32 6, i32 70, i32 134, i32 7, i32 71, i32 135, i32 8, i32 72, i32 136, i32 9, i32 73, i32 137, i32 10, i32 74, i32 138, i32 11, i32 75, i32 139, i32 12, i32 76, i32 140, i32 13, i32 77, i32 141, i32 14, i32 78, i32 142, i32 15, i32 79, i32 143, i32 16, i32 80, i32 144, i32 17, i32 81, i32 145, i32 18, i32 82, i32 146, i32 19, i32 83, i32 147, i32 20, i32 84, i32 148, i32 21, i32 85, i32 149, i32 22, i32 86, i32 150, i32 23, i32 87, i32 151, i32 24, i32 88, i32 152, i32 25, i32 89, i32 153, i32 26, i32 90, i32 154, i32 27, i32 91, i32 155, i32 28, i32 92, i32 156, i32 29, i32 93, i32 157, i32 30, i32 94, i32 158, i32 31, i32 95, i32 159, i32 32, i32 96, i32 160, i32 33, i32 97, i32 161, i32 34, i32 98, i32 162, i32 35, i32 99, i32 163, i32 36, i32 100, i32 164, i32 37, i32 101, i32 165, i32 38, i32 102, i32 166, i32 39, i32 103, i32 167, i32 40, i32 104, i32 168, i32 41, i32 105, i32 169, i32 42, i32 106, i32 170, i32 43, i32 107, i32 171, i32 44, i32 108, i32 172, i32 45, i32 109, i32 173, i32 46, i32 110, i32 174, i32 47, i32 111, i32 175, i32 48, i32 112, i32 176, i32 49, i32 113, i32 177, i32 50, i32 114, i32 178, i32 51, i32 115, i32 179, i32 52, i32 116, i32 180, i32 53, i32 117, i32 181, i32 54, i32 118, i32 182, i32 55, i32 119, i32 183, i32 56, i32 120, i32 184, i32 57, i32 121, i32 185, i32 58, i32 122, i32 186, i32 59, i32 123, i32 187, i32 60, i32 124, i32 188, i32 61, i32 125, i32 189, i32 62, i32 126, i32 190, i32 63, i32 127, i32 191>3804 store <192 x i16> %interleaved.vec, ptr %out.vec, align 643805 ret void3806}3807