brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE3; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx  | FileCheck %s --check-prefixes=AVX4; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX25; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FP6; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FCP7; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX5128; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512-FCP9; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ10; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-FCP11; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW12; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW-FCP13; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512DQ-BW14; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-BW-FCP15 16; These patterns are produced by LoopVectorizer for interleaved stores.17 18define void @store_i16_stride4_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %out.vec) nounwind {19; SSE-LABEL: store_i16_stride4_vf2:20; SSE:       # %bb.0:21; SSE-NEXT:    movdqa (%rdi), %xmm022; SSE-NEXT:    movdqa (%rdx), %xmm123; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]24; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]25; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]26; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]27; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,1,3,4,5,6,7]28; SSE-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,5,7]29; SSE-NEXT:    movdqa %xmm0, (%r8)30; SSE-NEXT:    retq31;32; AVX-LABEL: store_i16_stride4_vf2:33; AVX:       # %bb.0:34; AVX-NEXT:    vmovdqa (%rdi), %xmm035; AVX-NEXT:    vmovdqa (%rdx), %xmm136; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]37; AVX-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]38; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]39; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15]40; AVX-NEXT:    vmovdqa %xmm0, (%r8)41; AVX-NEXT:    retq42;43; AVX2-LABEL: store_i16_stride4_vf2:44; AVX2:       # %bb.0:45; AVX2-NEXT:    vmovdqa (%rdi), %xmm046; AVX2-NEXT:    vmovdqa (%rdx), %xmm147; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]48; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]49; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]50; AVX2-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15]51; AVX2-NEXT:    vmovdqa %xmm0, (%r8)52; AVX2-NEXT:    retq53;54; AVX2-FP-LABEL: store_i16_stride4_vf2:55; AVX2-FP:       # %bb.0:56; AVX2-FP-NEXT:    vmovdqa (%rdi), %xmm057; AVX2-FP-NEXT:    vmovdqa (%rdx), %xmm158; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]59; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]60; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]61; AVX2-FP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15]62; AVX2-FP-NEXT:    vmovdqa %xmm0, (%r8)63; AVX2-FP-NEXT:    retq64;65; AVX2-FCP-LABEL: store_i16_stride4_vf2:66; AVX2-FCP:       # %bb.0:67; AVX2-FCP-NEXT:    vmovdqa (%rdi), %xmm068; AVX2-FCP-NEXT:    vmovdqa (%rdx), %xmm169; AVX2-FCP-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]70; AVX2-FCP-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]71; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]72; AVX2-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15]73; AVX2-FCP-NEXT:    vmovdqa %xmm0, (%r8)74; AVX2-FCP-NEXT:    retq75;76; AVX512-LABEL: store_i16_stride4_vf2:77; AVX512:       # %bb.0:78; AVX512-NEXT:    vmovdqa (%rdi), %xmm079; AVX512-NEXT:    vmovdqa (%rdx), %xmm180; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]81; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]82; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]83; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15]84; AVX512-NEXT:    vmovdqa %xmm0, (%r8)85; AVX512-NEXT:    retq86;87; AVX512-FCP-LABEL: store_i16_stride4_vf2:88; AVX512-FCP:       # %bb.0:89; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm090; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm191; AVX512-FCP-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]92; AVX512-FCP-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]93; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]94; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15]95; AVX512-FCP-NEXT:    vmovdqa %xmm0, (%r8)96; AVX512-FCP-NEXT:    retq97;98; AVX512DQ-LABEL: store_i16_stride4_vf2:99; AVX512DQ:       # %bb.0:100; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0101; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm1102; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]103; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]104; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]105; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15]106; AVX512DQ-NEXT:    vmovdqa %xmm0, (%r8)107; AVX512DQ-NEXT:    retq108;109; AVX512DQ-FCP-LABEL: store_i16_stride4_vf2:110; AVX512DQ-FCP:       # %bb.0:111; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm0112; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm1113; AVX512DQ-FCP-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]114; AVX512DQ-FCP-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]115; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]116; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15]117; AVX512DQ-FCP-NEXT:    vmovdqa %xmm0, (%r8)118; AVX512DQ-FCP-NEXT:    retq119;120; AVX512BW-LABEL: store_i16_stride4_vf2:121; AVX512BW:       # %bb.0:122; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0123; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm1124; AVX512BW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]125; AVX512BW-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]126; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]127; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15]128; AVX512BW-NEXT:    vmovdqa %xmm0, (%r8)129; AVX512BW-NEXT:    retq130;131; AVX512BW-FCP-LABEL: store_i16_stride4_vf2:132; AVX512BW-FCP:       # %bb.0:133; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %xmm0134; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %xmm1135; AVX512BW-FCP-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]136; AVX512BW-FCP-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]137; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} xmm2 = [0,2,8,10,1,3,9,11]138; AVX512BW-FCP-NEXT:    vpermi2w %xmm1, %xmm0, %xmm2139; AVX512BW-FCP-NEXT:    vmovdqa %xmm2, (%r8)140; AVX512BW-FCP-NEXT:    retq141;142; AVX512DQ-BW-LABEL: store_i16_stride4_vf2:143; AVX512DQ-BW:       # %bb.0:144; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %xmm0145; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %xmm1146; AVX512DQ-BW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]147; AVX512DQ-BW-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]148; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]149; AVX512DQ-BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15]150; AVX512DQ-BW-NEXT:    vmovdqa %xmm0, (%r8)151; AVX512DQ-BW-NEXT:    retq152;153; AVX512DQ-BW-FCP-LABEL: store_i16_stride4_vf2:154; AVX512DQ-BW-FCP:       # %bb.0:155; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %xmm0156; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %xmm1157; AVX512DQ-BW-FCP-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]158; AVX512DQ-BW-FCP-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]159; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} xmm2 = [0,2,8,10,1,3,9,11]160; AVX512DQ-BW-FCP-NEXT:    vpermi2w %xmm1, %xmm0, %xmm2161; AVX512DQ-BW-FCP-NEXT:    vmovdqa %xmm2, (%r8)162; AVX512DQ-BW-FCP-NEXT:    retq163  %in.vec0 = load <2 x i16>, ptr %in.vecptr0, align 64164  %in.vec1 = load <2 x i16>, ptr %in.vecptr1, align 64165  %in.vec2 = load <2 x i16>, ptr %in.vecptr2, align 64166  %in.vec3 = load <2 x i16>, ptr %in.vecptr3, align 64167  %1 = shufflevector <2 x i16> %in.vec0, <2 x i16> %in.vec1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>168  %2 = shufflevector <2 x i16> %in.vec2, <2 x i16> %in.vec3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>169  %3 = shufflevector <4 x i16> %1, <4 x i16> %2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>170  %interleaved.vec = shufflevector <8 x i16> %3, <8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>171  store <8 x i16> %interleaved.vec, ptr %out.vec, align 64172  ret void173}174 175define void @store_i16_stride4_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %out.vec) nounwind {176; SSE-LABEL: store_i16_stride4_vf4:177; SSE:       # %bb.0:178; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero179; SSE-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero180; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]181; SSE-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero182; SSE-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero183; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]184; SSE-NEXT:    movdqa %xmm0, %xmm2185; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]186; SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]187; SSE-NEXT:    movdqa %xmm0, 16(%r8)188; SSE-NEXT:    movdqa %xmm2, (%r8)189; SSE-NEXT:    retq190;191; AVX-LABEL: store_i16_stride4_vf4:192; AVX:       # %bb.0:193; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero194; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero195; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm1[0],xmm0[0]196; AVX-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero197; AVX-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero198; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm4[0],xmm3[0]199; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]200; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]201; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]202; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm5[3,1,2,3]203; AVX-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[2,0,3,1,4,5,6,7]204; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]205; AVX-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[2,0,3,1,4,5,6,7]206; AVX-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]207; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0208; AVX-NEXT:    vmovaps %ymm0, (%r8)209; AVX-NEXT:    vzeroupper210; AVX-NEXT:    retq211;212; AVX2-LABEL: store_i16_stride4_vf4:213; AVX2:       # %bb.0:214; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero215; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero216; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]217; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero218; AVX2-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero219; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]220; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm2221; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0222; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,8,9,0,1,8,9,2,3,10,11,2,3,10,11,20,21,28,29,20,21,28,29,22,23,30,31,22,23,30,31]223; AVX2-NEXT:    vpshufb %ymm1, %ymm0, %ymm0224; AVX2-NEXT:    vpshufb %ymm1, %ymm2, %ymm1225; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6],ymm1[7]226; AVX2-NEXT:    vmovdqa %ymm0, (%r8)227; AVX2-NEXT:    vzeroupper228; AVX2-NEXT:    retq229;230; AVX2-FP-LABEL: store_i16_stride4_vf4:231; AVX2-FP:       # %bb.0:232; AVX2-FP-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero233; AVX2-FP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero234; AVX2-FP-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]235; AVX2-FP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero236; AVX2-FP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero237; AVX2-FP-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]238; AVX2-FP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm2239; AVX2-FP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0240; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,8,9,0,1,8,9,2,3,10,11,2,3,10,11,20,21,28,29,20,21,28,29,22,23,30,31,22,23,30,31]241; AVX2-FP-NEXT:    vpshufb %ymm1, %ymm0, %ymm0242; AVX2-FP-NEXT:    vpshufb %ymm1, %ymm2, %ymm1243; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6],ymm1[7]244; AVX2-FP-NEXT:    vmovdqa %ymm0, (%r8)245; AVX2-FP-NEXT:    vzeroupper246; AVX2-FP-NEXT:    retq247;248; AVX2-FCP-LABEL: store_i16_stride4_vf4:249; AVX2-FCP:       # %bb.0:250; AVX2-FCP-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero251; AVX2-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero252; AVX2-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]253; AVX2-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero254; AVX2-FCP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero255; AVX2-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]256; AVX2-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0257; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [0,2,4,6,1,3,5,7]258; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm1, %ymm0259; AVX2-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,2,3,6,7,10,11,14,15,16,17,20,21,24,25,28,29,18,19,22,23,26,27,30,31]260; AVX2-FCP-NEXT:    vmovdqa %ymm0, (%r8)261; AVX2-FCP-NEXT:    vzeroupper262; AVX2-FCP-NEXT:    retq263;264; AVX512-LABEL: store_i16_stride4_vf4:265; AVX512:       # %bb.0:266; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero267; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero268; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]269; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero270; AVX512-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero271; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]272; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm2273; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0274; AVX512-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,8,9,0,1,8,9,2,3,10,11,2,3,10,11,20,21,28,29,20,21,28,29,22,23,30,31,22,23,30,31]275; AVX512-NEXT:    vpshufb %ymm1, %ymm0, %ymm0276; AVX512-NEXT:    vpshufb %ymm1, %ymm2, %ymm1277; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6],ymm1[7]278; AVX512-NEXT:    vmovdqa %ymm0, (%r8)279; AVX512-NEXT:    vzeroupper280; AVX512-NEXT:    retq281;282; AVX512-FCP-LABEL: store_i16_stride4_vf4:283; AVX512-FCP:       # %bb.0:284; AVX512-FCP-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero285; AVX512-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero286; AVX512-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]287; AVX512-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero288; AVX512-FCP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero289; AVX512-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]290; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm2 = [0,2,8,10,1,3,9,11]291; AVX512-FCP-NEXT:    vpermi2d %ymm1, %ymm0, %ymm2292; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[0,1,4,5,8,9,12,13,2,3,6,7,10,11,14,15,16,17,20,21,24,25,28,29,18,19,22,23,26,27,30,31]293; AVX512-FCP-NEXT:    vmovdqa %ymm0, (%r8)294; AVX512-FCP-NEXT:    vzeroupper295; AVX512-FCP-NEXT:    retq296;297; AVX512DQ-LABEL: store_i16_stride4_vf4:298; AVX512DQ:       # %bb.0:299; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero300; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero301; AVX512DQ-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]302; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero303; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero304; AVX512DQ-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]305; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm2306; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0307; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,8,9,0,1,8,9,2,3,10,11,2,3,10,11,20,21,28,29,20,21,28,29,22,23,30,31,22,23,30,31]308; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm0309; AVX512DQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm1310; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3,4],ymm1[5],ymm0[6],ymm1[7]311; AVX512DQ-NEXT:    vmovdqa %ymm0, (%r8)312; AVX512DQ-NEXT:    vzeroupper313; AVX512DQ-NEXT:    retq314;315; AVX512DQ-FCP-LABEL: store_i16_stride4_vf4:316; AVX512DQ-FCP:       # %bb.0:317; AVX512DQ-FCP-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero318; AVX512DQ-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero319; AVX512DQ-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]320; AVX512DQ-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero321; AVX512DQ-FCP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero322; AVX512DQ-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]323; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm2 = [0,2,8,10,1,3,9,11]324; AVX512DQ-FCP-NEXT:    vpermi2d %ymm1, %ymm0, %ymm2325; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[0,1,4,5,8,9,12,13,2,3,6,7,10,11,14,15,16,17,20,21,24,25,28,29,18,19,22,23,26,27,30,31]326; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, (%r8)327; AVX512DQ-FCP-NEXT:    vzeroupper328; AVX512DQ-FCP-NEXT:    retq329;330; AVX512BW-LABEL: store_i16_stride4_vf4:331; AVX512BW:       # %bb.0:332; AVX512BW-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero333; AVX512BW-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero334; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]335; AVX512BW-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero336; AVX512BW-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero337; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]338; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm2 = [0,4,16,20,1,5,17,21,2,6,18,22,3,7,19,23]339; AVX512BW-NEXT:    vpermi2w %ymm1, %ymm0, %ymm2340; AVX512BW-NEXT:    vmovdqa %ymm2, (%r8)341; AVX512BW-NEXT:    vzeroupper342; AVX512BW-NEXT:    retq343;344; AVX512BW-FCP-LABEL: store_i16_stride4_vf4:345; AVX512BW-FCP:       # %bb.0:346; AVX512BW-FCP-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero347; AVX512BW-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero348; AVX512BW-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]349; AVX512BW-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero350; AVX512BW-FCP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero351; AVX512BW-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]352; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm2 = [0,4,16,20,1,5,17,21,2,6,18,22,3,7,19,23]353; AVX512BW-FCP-NEXT:    vpermi2w %ymm1, %ymm0, %ymm2354; AVX512BW-FCP-NEXT:    vmovdqa %ymm2, (%r8)355; AVX512BW-FCP-NEXT:    vzeroupper356; AVX512BW-FCP-NEXT:    retq357;358; AVX512DQ-BW-LABEL: store_i16_stride4_vf4:359; AVX512DQ-BW:       # %bb.0:360; AVX512DQ-BW-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero361; AVX512DQ-BW-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero362; AVX512DQ-BW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]363; AVX512DQ-BW-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero364; AVX512DQ-BW-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero365; AVX512DQ-BW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]366; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm2 = [0,4,16,20,1,5,17,21,2,6,18,22,3,7,19,23]367; AVX512DQ-BW-NEXT:    vpermi2w %ymm1, %ymm0, %ymm2368; AVX512DQ-BW-NEXT:    vmovdqa %ymm2, (%r8)369; AVX512DQ-BW-NEXT:    vzeroupper370; AVX512DQ-BW-NEXT:    retq371;372; AVX512DQ-BW-FCP-LABEL: store_i16_stride4_vf4:373; AVX512DQ-BW-FCP:       # %bb.0:374; AVX512DQ-BW-FCP-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero375; AVX512DQ-BW-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero376; AVX512DQ-BW-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]377; AVX512DQ-BW-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero378; AVX512DQ-BW-FCP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero379; AVX512DQ-BW-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]380; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm2 = [0,4,16,20,1,5,17,21,2,6,18,22,3,7,19,23]381; AVX512DQ-BW-FCP-NEXT:    vpermi2w %ymm1, %ymm0, %ymm2382; AVX512DQ-BW-FCP-NEXT:    vmovdqa %ymm2, (%r8)383; AVX512DQ-BW-FCP-NEXT:    vzeroupper384; AVX512DQ-BW-FCP-NEXT:    retq385  %in.vec0 = load <4 x i16>, ptr %in.vecptr0, align 64386  %in.vec1 = load <4 x i16>, ptr %in.vecptr1, align 64387  %in.vec2 = load <4 x i16>, ptr %in.vecptr2, align 64388  %in.vec3 = load <4 x i16>, ptr %in.vecptr3, align 64389  %1 = shufflevector <4 x i16> %in.vec0, <4 x i16> %in.vec1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>390  %2 = shufflevector <4 x i16> %in.vec2, <4 x i16> %in.vec3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>391  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>392  %interleaved.vec = shufflevector <16 x i16> %3, <16 x i16> poison, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>393  store <16 x i16> %interleaved.vec, ptr %out.vec, align 64394  ret void395}396 397define void @store_i16_stride4_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %out.vec) nounwind {398; SSE-LABEL: store_i16_stride4_vf8:399; SSE:       # %bb.0:400; SSE-NEXT:    movdqa (%rdi), %xmm0401; SSE-NEXT:    movdqa (%rsi), %xmm1402; SSE-NEXT:    movdqa (%rdx), %xmm2403; SSE-NEXT:    movdqa (%rcx), %xmm3404; SSE-NEXT:    movdqa %xmm2, %xmm4405; SSE-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]406; SSE-NEXT:    movdqa %xmm0, %xmm5407; SSE-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]408; SSE-NEXT:    movdqa %xmm5, %xmm6409; SSE-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]410; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]411; SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]412; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]413; SSE-NEXT:    movdqa %xmm0, %xmm1414; SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]415; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]416; SSE-NEXT:    movdqa %xmm0, 32(%r8)417; SSE-NEXT:    movdqa %xmm1, 48(%r8)418; SSE-NEXT:    movdqa %xmm5, 16(%r8)419; SSE-NEXT:    movdqa %xmm6, (%r8)420; SSE-NEXT:    retq421;422; AVX-LABEL: store_i16_stride4_vf8:423; AVX:       # %bb.0:424; AVX-NEXT:    vmovdqa (%rdi), %xmm0425; AVX-NEXT:    vmovdqa (%rsi), %xmm1426; AVX-NEXT:    vmovdqa (%rdx), %xmm2427; AVX-NEXT:    vmovdqa (%rcx), %xmm3428; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]429; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]430; AVX-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]431; AVX-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]432; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm4, %ymm4433; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]434; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]435; AVX-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]436; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]437; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0438; AVX-NEXT:    vmovaps %ymm0, 32(%r8)439; AVX-NEXT:    vmovaps %ymm4, (%r8)440; AVX-NEXT:    vzeroupper441; AVX-NEXT:    retq442;443; AVX2-LABEL: store_i16_stride4_vf8:444; AVX2:       # %bb.0:445; AVX2-NEXT:    vmovdqa (%rdi), %xmm0446; AVX2-NEXT:    vmovdqa (%rdx), %xmm1447; AVX2-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0448; AVX2-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1449; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[0,2,0,2]450; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [u,u,u,u,0,1,8,9,u,u,u,u,2,3,10,11,u,u,u,u,4,5,12,13,u,u,u,u,6,7,14,15]451; AVX2-NEXT:    vpshufb %ymm3, %ymm2, %ymm2452; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[0,2,0,2]453; AVX2-NEXT:    vpmovsxdq {{.*#+}} ymm5 = [151519488,185205506,218891524,252577542]454; AVX2-NEXT:    vpshufb %ymm5, %ymm4, %ymm4455; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2],ymm2[3],ymm4[4],ymm2[5],ymm4[6],ymm2[7]456; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]457; AVX2-NEXT:    vpshufb %ymm3, %ymm1, %ymm1458; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,3,1,3]459; AVX2-NEXT:    vpshufb %ymm5, %ymm0, %ymm0460; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]461; AVX2-NEXT:    vmovdqa %ymm0, 32(%r8)462; AVX2-NEXT:    vmovdqa %ymm2, (%r8)463; AVX2-NEXT:    vzeroupper464; AVX2-NEXT:    retq465;466; AVX2-FP-LABEL: store_i16_stride4_vf8:467; AVX2-FP:       # %bb.0:468; AVX2-FP-NEXT:    vmovdqa (%rdi), %xmm0469; AVX2-FP-NEXT:    vmovdqa (%rdx), %xmm1470; AVX2-FP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0471; AVX2-FP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1472; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[0,2,0,2]473; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm3 = [u,u,u,u,0,1,8,9,u,u,u,u,2,3,10,11,u,u,u,u,4,5,12,13,u,u,u,u,6,7,14,15]474; AVX2-FP-NEXT:    vpshufb %ymm3, %ymm2, %ymm2475; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[0,2,0,2]476; AVX2-FP-NEXT:    vpmovsxdq {{.*#+}} ymm5 = [151519488,185205506,218891524,252577542]477; AVX2-FP-NEXT:    vpshufb %ymm5, %ymm4, %ymm4478; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2],ymm2[3],ymm4[4],ymm2[5],ymm4[6],ymm2[7]479; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]480; AVX2-FP-NEXT:    vpshufb %ymm3, %ymm1, %ymm1481; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,3,1,3]482; AVX2-FP-NEXT:    vpshufb %ymm5, %ymm0, %ymm0483; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]484; AVX2-FP-NEXT:    vmovdqa %ymm0, 32(%r8)485; AVX2-FP-NEXT:    vmovdqa %ymm2, (%r8)486; AVX2-FP-NEXT:    vzeroupper487; AVX2-FP-NEXT:    retq488;489; AVX2-FCP-LABEL: store_i16_stride4_vf8:490; AVX2-FCP:       # %bb.0:491; AVX2-FCP-NEXT:    vmovdqa (%rdi), %xmm0492; AVX2-FCP-NEXT:    vmovdqa (%rdx), %xmm1493; AVX2-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0494; AVX2-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1495; AVX2-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[0,2,0,2]496; AVX2-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [u,u,u,u,0,1,8,9,u,u,u,u,2,3,10,11,u,u,u,u,4,5,12,13,u,u,u,u,6,7,14,15]497; AVX2-FCP-NEXT:    vpshufb %ymm3, %ymm2, %ymm2498; AVX2-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[0,2,0,2]499; AVX2-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm5 = [151519488,185205506,218891524,252577542]500; AVX2-FCP-NEXT:    vpshufb %ymm5, %ymm4, %ymm4501; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2],ymm2[3],ymm4[4],ymm2[5],ymm4[6],ymm2[7]502; AVX2-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]503; AVX2-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm1504; AVX2-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,3,1,3]505; AVX2-FCP-NEXT:    vpshufb %ymm5, %ymm0, %ymm0506; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]507; AVX2-FCP-NEXT:    vmovdqa %ymm0, 32(%r8)508; AVX2-FCP-NEXT:    vmovdqa %ymm2, (%r8)509; AVX2-FCP-NEXT:    vzeroupper510; AVX2-FCP-NEXT:    retq511;512; AVX512-LABEL: store_i16_stride4_vf8:513; AVX512:       # %bb.0:514; AVX512-NEXT:    vmovdqa (%rdi), %xmm0515; AVX512-NEXT:    vmovdqa (%rdx), %xmm1516; AVX512-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1517; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[1,3,1,3]518; AVX512-NEXT:    vmovdqa {{.*#+}} ymm3 = [u,u,u,u,0,1,8,9,u,u,u,u,2,3,10,11,u,u,u,u,4,5,12,13,u,u,u,u,6,7,14,15]519; AVX512-NEXT:    vpshufb %ymm3, %ymm2, %ymm2520; AVX512-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0521; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[1,3,1,3]522; AVX512-NEXT:    vpmovsxdq {{.*#+}} ymm5 = [151519488,185205506,218891524,252577542]523; AVX512-NEXT:    vpshufb %ymm5, %ymm4, %ymm4524; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2],ymm2[3],ymm4[4],ymm2[5],ymm4[6],ymm2[7]525; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,0,2]526; AVX512-NEXT:    vpshufb %ymm3, %ymm1, %ymm1527; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,0,2]528; AVX512-NEXT:    vpshufb %ymm5, %ymm0, %ymm0529; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]530; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0531; AVX512-NEXT:    vmovdqa64 %zmm0, (%r8)532; AVX512-NEXT:    vzeroupper533; AVX512-NEXT:    retq534;535; AVX512-FCP-LABEL: store_i16_stride4_vf8:536; AVX512-FCP:       # %bb.0:537; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm0538; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm1539; AVX512-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1540; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[1,3,1,3]541; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [u,u,u,u,0,1,8,9,u,u,u,u,2,3,10,11,u,u,u,u,4,5,12,13,u,u,u,u,6,7,14,15]542; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm2, %ymm2543; AVX512-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0544; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[1,3,1,3]545; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm5 = [151519488,185205506,218891524,252577542]546; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm4, %ymm4547; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2],ymm2[3],ymm4[4],ymm2[5],ymm4[6],ymm2[7]548; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,0,2]549; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm1550; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,0,2]551; AVX512-FCP-NEXT:    vpshufb %ymm5, %ymm0, %ymm0552; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]553; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0554; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, (%r8)555; AVX512-FCP-NEXT:    vzeroupper556; AVX512-FCP-NEXT:    retq557;558; AVX512DQ-LABEL: store_i16_stride4_vf8:559; AVX512DQ:       # %bb.0:560; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0561; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm1562; AVX512DQ-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1563; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[1,3,1,3]564; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [u,u,u,u,0,1,8,9,u,u,u,u,2,3,10,11,u,u,u,u,4,5,12,13,u,u,u,u,6,7,14,15]565; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm2, %ymm2566; AVX512DQ-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0567; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[1,3,1,3]568; AVX512DQ-NEXT:    vpmovsxdq {{.*#+}} ymm5 = [151519488,185205506,218891524,252577542]569; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm4, %ymm4570; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2],ymm2[3],ymm4[4],ymm2[5],ymm4[6],ymm2[7]571; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,0,2]572; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm1, %ymm1573; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,0,2]574; AVX512DQ-NEXT:    vpshufb %ymm5, %ymm0, %ymm0575; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]576; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0577; AVX512DQ-NEXT:    vmovdqa64 %zmm0, (%r8)578; AVX512DQ-NEXT:    vzeroupper579; AVX512DQ-NEXT:    retq580;581; AVX512DQ-FCP-LABEL: store_i16_stride4_vf8:582; AVX512DQ-FCP:       # %bb.0:583; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm0584; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm1585; AVX512DQ-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1586; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[1,3,1,3]587; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [u,u,u,u,0,1,8,9,u,u,u,u,2,3,10,11,u,u,u,u,4,5,12,13,u,u,u,u,6,7,14,15]588; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm2, %ymm2589; AVX512DQ-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0590; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[1,3,1,3]591; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm5 = [151519488,185205506,218891524,252577542]592; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm4, %ymm4593; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2],ymm2[3],ymm4[4],ymm2[5],ymm4[6],ymm2[7]594; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,0,2]595; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm1596; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,0,2]597; AVX512DQ-FCP-NEXT:    vpshufb %ymm5, %ymm0, %ymm0598; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]599; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0600; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, (%r8)601; AVX512DQ-FCP-NEXT:    vzeroupper602; AVX512DQ-FCP-NEXT:    retq603;604; AVX512BW-LABEL: store_i16_stride4_vf8:605; AVX512BW:       # %bb.0:606; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0607; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm1608; AVX512BW-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1609; AVX512BW-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0610; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0611; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [0,8,16,24,1,9,17,25,2,10,18,26,3,11,19,27,4,12,20,28,5,13,21,29,6,14,22,30,7,15,23,31]612; AVX512BW-NEXT:    vpermw %zmm0, %zmm1, %zmm0613; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%r8)614; AVX512BW-NEXT:    vzeroupper615; AVX512BW-NEXT:    retq616;617; AVX512BW-FCP-LABEL: store_i16_stride4_vf8:618; AVX512BW-FCP:       # %bb.0:619; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %xmm0620; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %xmm1621; AVX512BW-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1622; AVX512BW-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0623; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0624; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [0,8,16,24,1,9,17,25,2,10,18,26,3,11,19,27,4,12,20,28,5,13,21,29,6,14,22,30,7,15,23,31]625; AVX512BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm0626; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, (%r8)627; AVX512BW-FCP-NEXT:    vzeroupper628; AVX512BW-FCP-NEXT:    retq629;630; AVX512DQ-BW-LABEL: store_i16_stride4_vf8:631; AVX512DQ-BW:       # %bb.0:632; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %xmm0633; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %xmm1634; AVX512DQ-BW-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1635; AVX512DQ-BW-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0636; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0637; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [0,8,16,24,1,9,17,25,2,10,18,26,3,11,19,27,4,12,20,28,5,13,21,29,6,14,22,30,7,15,23,31]638; AVX512DQ-BW-NEXT:    vpermw %zmm0, %zmm1, %zmm0639; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, (%r8)640; AVX512DQ-BW-NEXT:    vzeroupper641; AVX512DQ-BW-NEXT:    retq642;643; AVX512DQ-BW-FCP-LABEL: store_i16_stride4_vf8:644; AVX512DQ-BW-FCP:       # %bb.0:645; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %xmm0646; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %xmm1647; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1648; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm0649; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0650; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [0,8,16,24,1,9,17,25,2,10,18,26,3,11,19,27,4,12,20,28,5,13,21,29,6,14,22,30,7,15,23,31]651; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm0, %zmm1, %zmm0652; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, (%r8)653; AVX512DQ-BW-FCP-NEXT:    vzeroupper654; AVX512DQ-BW-FCP-NEXT:    retq655  %in.vec0 = load <8 x i16>, ptr %in.vecptr0, align 64656  %in.vec1 = load <8 x i16>, ptr %in.vecptr1, align 64657  %in.vec2 = load <8 x i16>, ptr %in.vecptr2, align 64658  %in.vec3 = load <8 x i16>, ptr %in.vecptr3, align 64659  %1 = shufflevector <8 x i16> %in.vec0, <8 x i16> %in.vec1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>660  %2 = shufflevector <8 x i16> %in.vec2, <8 x i16> %in.vec3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>661  %3 = shufflevector <16 x i16> %1, <16 x i16> %2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>662  %interleaved.vec = shufflevector <32 x i16> %3, <32 x i16> poison, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>663  store <32 x i16> %interleaved.vec, ptr %out.vec, align 64664  ret void665}666 667define void @store_i16_stride4_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %out.vec) nounwind {668; SSE-LABEL: store_i16_stride4_vf16:669; SSE:       # %bb.0:670; SSE-NEXT:    movdqa (%rdi), %xmm0671; SSE-NEXT:    movdqa 16(%rdi), %xmm1672; SSE-NEXT:    movdqa (%rsi), %xmm5673; SSE-NEXT:    movdqa 16(%rsi), %xmm6674; SSE-NEXT:    movdqa (%rdx), %xmm7675; SSE-NEXT:    movdqa 16(%rdx), %xmm4676; SSE-NEXT:    movdqa (%rcx), %xmm8677; SSE-NEXT:    movdqa 16(%rcx), %xmm9678; SSE-NEXT:    movdqa %xmm7, %xmm10679; SSE-NEXT:    punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]680; SSE-NEXT:    movdqa %xmm0, %xmm2681; SSE-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]682; SSE-NEXT:    movdqa %xmm2, %xmm3683; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm10[2],xmm3[3],xmm10[3]684; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1]685; SSE-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7]686; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7]687; SSE-NEXT:    movdqa %xmm0, %xmm5688; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm7[2],xmm5[3],xmm7[3]689; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1]690; SSE-NEXT:    movdqa %xmm4, %xmm7691; SSE-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]692; SSE-NEXT:    movdqa %xmm1, %xmm8693; SSE-NEXT:    punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]694; SSE-NEXT:    movdqa %xmm8, %xmm10695; SSE-NEXT:    punpckhdq {{.*#+}} xmm10 = xmm10[2],xmm7[2],xmm10[3],xmm7[3]696; SSE-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]697; SSE-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm9[4],xmm4[5],xmm9[5],xmm4[6],xmm9[6],xmm4[7],xmm9[7]698; SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]699; SSE-NEXT:    movdqa %xmm1, %xmm6700; SSE-NEXT:    punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm4[2],xmm6[3],xmm4[3]701; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]702; SSE-NEXT:    movdqa %xmm1, 96(%r8)703; SSE-NEXT:    movdqa %xmm6, 112(%r8)704; SSE-NEXT:    movdqa %xmm8, 64(%r8)705; SSE-NEXT:    movdqa %xmm10, 80(%r8)706; SSE-NEXT:    movdqa %xmm0, 32(%r8)707; SSE-NEXT:    movdqa %xmm5, 48(%r8)708; SSE-NEXT:    movdqa %xmm2, (%r8)709; SSE-NEXT:    movdqa %xmm3, 16(%r8)710; SSE-NEXT:    retq711;712; AVX-LABEL: store_i16_stride4_vf16:713; AVX:       # %bb.0:714; AVX-NEXT:    vmovdqa (%rcx), %xmm5715; AVX-NEXT:    vmovdqa 16(%rcx), %xmm0716; AVX-NEXT:    vmovdqa (%rdx), %xmm6717; AVX-NEXT:    vmovdqa 16(%rdx), %xmm1718; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]719; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[0,0,1,1]720; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]721; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2722; AVX-NEXT:    vmovdqa (%rsi), %xmm7723; AVX-NEXT:    vmovdqa 16(%rsi), %xmm3724; AVX-NEXT:    vmovdqa (%rdi), %xmm8725; AVX-NEXT:    vmovdqa 16(%rdi), %xmm4726; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]727; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm10 = xmm9[0],zero,xmm9[1],zero728; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]729; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm10, %ymm9730; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm9[0],ymm2[1],ymm9[2],ymm2[3],ymm9[4],ymm2[5],ymm9[6],ymm2[7]731; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]732; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm5[0,0,1,1]733; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]734; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5735; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]736; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero737; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]738; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm7, %ymm6739; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7]740; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]741; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[0,0,1,1]742; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]743; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm7, %ymm6744; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]745; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm7[0],zero,xmm7[1],zero746; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]747; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm8, %ymm7748; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0],ymm6[1],ymm7[2],ymm6[3],ymm7[4],ymm6[5],ymm7[6],ymm6[7]749; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]750; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]751; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]752; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0753; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]754; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero755; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]756; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1757; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]758; AVX-NEXT:    vmovaps %ymm0, 64(%r8)759; AVX-NEXT:    vmovaps %ymm6, 96(%r8)760; AVX-NEXT:    vmovaps %ymm5, 32(%r8)761; AVX-NEXT:    vmovaps %ymm2, (%r8)762; AVX-NEXT:    vzeroupper763; AVX-NEXT:    retq764;765; AVX2-LABEL: store_i16_stride4_vf16:766; AVX2:       # %bb.0:767; AVX2-NEXT:    vmovdqa (%rcx), %xmm5768; AVX2-NEXT:    vmovdqa 16(%rcx), %xmm0769; AVX2-NEXT:    vmovdqa (%rdx), %xmm6770; AVX2-NEXT:    vmovdqa 16(%rdx), %xmm1771; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]772; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[0,0,1,1]773; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]774; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2775; AVX2-NEXT:    vmovdqa (%rsi), %xmm7776; AVX2-NEXT:    vmovdqa 16(%rsi), %xmm3777; AVX2-NEXT:    vmovdqa (%rdi), %xmm8778; AVX2-NEXT:    vmovdqa 16(%rdi), %xmm4779; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]780; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm10 = xmm9[0],zero,xmm9[1],zero781; AVX2-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]782; AVX2-NEXT:    vinserti128 $1, %xmm9, %ymm10, %ymm9783; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm9[0],ymm2[1],ymm9[2],ymm2[3],ymm9[4],ymm2[5],ymm9[6],ymm2[7]784; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]785; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm5[0,0,1,1]786; AVX2-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]787; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm6, %ymm5788; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]789; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero790; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]791; AVX2-NEXT:    vinserti128 $1, %xmm6, %ymm7, %ymm6792; AVX2-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7]793; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]794; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[0,0,1,1]795; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]796; AVX2-NEXT:    vinserti128 $1, %xmm6, %ymm7, %ymm6797; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]798; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm7[0],zero,xmm7[1],zero799; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]800; AVX2-NEXT:    vinserti128 $1, %xmm7, %ymm8, %ymm7801; AVX2-NEXT:    vpblendd {{.*#+}} ymm6 = ymm7[0],ymm6[1],ymm7[2],ymm6[3],ymm7[4],ymm6[5],ymm7[6],ymm6[7]802; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]803; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]804; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]805; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0806; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]807; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero808; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]809; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm3, %ymm1810; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]811; AVX2-NEXT:    vmovdqa %ymm0, 64(%r8)812; AVX2-NEXT:    vmovdqa %ymm6, 96(%r8)813; AVX2-NEXT:    vmovdqa %ymm5, 32(%r8)814; AVX2-NEXT:    vmovdqa %ymm2, (%r8)815; AVX2-NEXT:    vzeroupper816; AVX2-NEXT:    retq817;818; AVX2-FP-LABEL: store_i16_stride4_vf16:819; AVX2-FP:       # %bb.0:820; AVX2-FP-NEXT:    vmovdqa (%rcx), %xmm5821; AVX2-FP-NEXT:    vmovdqa 16(%rcx), %xmm0822; AVX2-FP-NEXT:    vmovdqa (%rdx), %xmm6823; AVX2-FP-NEXT:    vmovdqa 16(%rdx), %xmm1824; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]825; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[0,0,1,1]826; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]827; AVX2-FP-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2828; AVX2-FP-NEXT:    vmovdqa (%rsi), %xmm7829; AVX2-FP-NEXT:    vmovdqa 16(%rsi), %xmm3830; AVX2-FP-NEXT:    vmovdqa (%rdi), %xmm8831; AVX2-FP-NEXT:    vmovdqa 16(%rdi), %xmm4832; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]833; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm10 = xmm9[0],zero,xmm9[1],zero834; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]835; AVX2-FP-NEXT:    vinserti128 $1, %xmm9, %ymm10, %ymm9836; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm9[0],ymm2[1],ymm9[2],ymm2[3],ymm9[4],ymm2[5],ymm9[6],ymm2[7]837; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]838; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm5[0,0,1,1]839; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]840; AVX2-FP-NEXT:    vinserti128 $1, %xmm5, %ymm6, %ymm5841; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]842; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero843; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]844; AVX2-FP-NEXT:    vinserti128 $1, %xmm6, %ymm7, %ymm6845; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4],ymm5[5],ymm6[6],ymm5[7]846; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]847; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[0,0,1,1]848; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]849; AVX2-FP-NEXT:    vinserti128 $1, %xmm6, %ymm7, %ymm6850; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]851; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm7[0],zero,xmm7[1],zero852; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]853; AVX2-FP-NEXT:    vinserti128 $1, %xmm7, %ymm8, %ymm7854; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm7[0],ymm6[1],ymm7[2],ymm6[3],ymm7[4],ymm6[5],ymm7[6],ymm6[7]855; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]856; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]857; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]858; AVX2-FP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0859; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]860; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero861; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]862; AVX2-FP-NEXT:    vinserti128 $1, %xmm1, %ymm3, %ymm1863; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]864; AVX2-FP-NEXT:    vmovdqa %ymm0, 64(%r8)865; AVX2-FP-NEXT:    vmovdqa %ymm6, 96(%r8)866; AVX2-FP-NEXT:    vmovdqa %ymm5, 32(%r8)867; AVX2-FP-NEXT:    vmovdqa %ymm2, (%r8)868; AVX2-FP-NEXT:    vzeroupper869; AVX2-FP-NEXT:    retq870;871; AVX2-FCP-LABEL: store_i16_stride4_vf16:872; AVX2-FCP:       # %bb.0:873; AVX2-FCP-NEXT:    vmovdqa (%rsi), %xmm2874; AVX2-FCP-NEXT:    vmovdqa 16(%rsi), %xmm0875; AVX2-FCP-NEXT:    vmovdqa (%rdi), %xmm3876; AVX2-FCP-NEXT:    vmovdqa 16(%rdi), %xmm1877; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]878; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero879; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]880; AVX2-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm4881; AVX2-FCP-NEXT:    vmovdqa (%rcx), %xmm5882; AVX2-FCP-NEXT:    vmovdqa 16(%rcx), %xmm6883; AVX2-FCP-NEXT:    vmovdqa (%rdx), %xmm7884; AVX2-FCP-NEXT:    vmovdqa 16(%rdx), %xmm8885; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]886; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [0,0,1,1,2,2,3,3]887; AVX2-FCP-NEXT:    vpermd %ymm9, %ymm10, %ymm9888; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0],ymm9[1],ymm4[2],ymm9[3],ymm4[4],ymm9[5],ymm4[6],ymm9[7]889; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]890; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero891; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]892; AVX2-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2893; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]894; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm10, %ymm3895; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2],ymm3[3],ymm2[4],ymm3[5],ymm2[6],ymm3[7]896; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]897; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero898; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,2,3,3]899; AVX2-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm5, %ymm3900; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]901; AVX2-FCP-NEXT:    vpermd %ymm5, %ymm10, %ymm5902; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4],ymm5[5],ymm3[6],ymm5[7]903; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]904; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero905; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]906; AVX2-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0907; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]908; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm10, %ymm1909; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]910; AVX2-FCP-NEXT:    vmovdqa %ymm0, 64(%r8)911; AVX2-FCP-NEXT:    vmovdqa %ymm3, 96(%r8)912; AVX2-FCP-NEXT:    vmovdqa %ymm2, 32(%r8)913; AVX2-FCP-NEXT:    vmovdqa %ymm4, (%r8)914; AVX2-FCP-NEXT:    vzeroupper915; AVX2-FCP-NEXT:    retq916;917; AVX512-LABEL: store_i16_stride4_vf16:918; AVX512:       # %bb.0:919; AVX512-NEXT:    vmovdqa (%rcx), %xmm0920; AVX512-NEXT:    vmovdqa 16(%rcx), %xmm1921; AVX512-NEXT:    vmovdqa (%rdx), %xmm2922; AVX512-NEXT:    vmovdqa 16(%rdx), %xmm3923; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]924; AVX512-NEXT:    vmovdqa (%rsi), %xmm5925; AVX512-NEXT:    vmovdqa 16(%rsi), %xmm6926; AVX512-NEXT:    vmovdqa (%rdi), %xmm7927; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm8928; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]929; AVX512-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [0,8,1,9,2,10,3,11]930; AVX512-NEXT:    vpermt2d %ymm4, %ymm10, %ymm9931; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]932; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]933; AVX512-NEXT:    vpermt2d %ymm1, %ymm10, %ymm3934; AVX512-NEXT:    vinserti64x4 $1, %ymm9, %zmm3, %zmm1935; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]936; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]937; AVX512-NEXT:    vpermt2d %ymm3, %ymm10, %ymm4938; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]939; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]940; AVX512-NEXT:    vpermt2d %ymm0, %ymm10, %ymm2941; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm0942; AVX512-NEXT:    vmovdqa64 %zmm0, (%r8)943; AVX512-NEXT:    vmovdqa64 %zmm1, 64(%r8)944; AVX512-NEXT:    vzeroupper945; AVX512-NEXT:    retq946;947; AVX512-FCP-LABEL: store_i16_stride4_vf16:948; AVX512-FCP:       # %bb.0:949; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm0950; AVX512-FCP-NEXT:    vmovdqa 16(%rcx), %xmm1951; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm2952; AVX512-FCP-NEXT:    vmovdqa 16(%rdx), %xmm3953; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]954; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm5955; AVX512-FCP-NEXT:    vmovdqa 16(%rsi), %xmm6956; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm7957; AVX512-FCP-NEXT:    vmovdqa 16(%rdi), %xmm8958; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]959; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [0,8,1,9,2,10,3,11]960; AVX512-FCP-NEXT:    vpermt2d %ymm4, %ymm10, %ymm9961; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]962; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]963; AVX512-FCP-NEXT:    vpermt2d %ymm1, %ymm10, %ymm3964; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm3, %zmm1965; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]966; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]967; AVX512-FCP-NEXT:    vpermt2d %ymm3, %ymm10, %ymm4968; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]969; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]970; AVX512-FCP-NEXT:    vpermt2d %ymm0, %ymm10, %ymm2971; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm0972; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, (%r8)973; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 64(%r8)974; AVX512-FCP-NEXT:    vzeroupper975; AVX512-FCP-NEXT:    retq976;977; AVX512DQ-LABEL: store_i16_stride4_vf16:978; AVX512DQ:       # %bb.0:979; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm0980; AVX512DQ-NEXT:    vmovdqa 16(%rcx), %xmm1981; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm2982; AVX512DQ-NEXT:    vmovdqa 16(%rdx), %xmm3983; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]984; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm5985; AVX512DQ-NEXT:    vmovdqa 16(%rsi), %xmm6986; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm7987; AVX512DQ-NEXT:    vmovdqa 16(%rdi), %xmm8988; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]989; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [0,8,1,9,2,10,3,11]990; AVX512DQ-NEXT:    vpermt2d %ymm4, %ymm10, %ymm9991; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]992; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]993; AVX512DQ-NEXT:    vpermt2d %ymm1, %ymm10, %ymm3994; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm9, %zmm3, %zmm1995; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]996; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]997; AVX512DQ-NEXT:    vpermt2d %ymm3, %ymm10, %ymm4998; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]999; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]1000; AVX512DQ-NEXT:    vpermt2d %ymm0, %ymm10, %ymm21001; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm01002; AVX512DQ-NEXT:    vmovdqa64 %zmm0, (%r8)1003; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 64(%r8)1004; AVX512DQ-NEXT:    vzeroupper1005; AVX512DQ-NEXT:    retq1006;1007; AVX512DQ-FCP-LABEL: store_i16_stride4_vf16:1008; AVX512DQ-FCP:       # %bb.0:1009; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm01010; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rcx), %xmm11011; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm21012; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rdx), %xmm31013; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]1014; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm51015; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rsi), %xmm61016; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm71017; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rdi), %xmm81018; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]1019; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [0,8,1,9,2,10,3,11]1020; AVX512DQ-FCP-NEXT:    vpermt2d %ymm4, %ymm10, %ymm91021; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]1022; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]1023; AVX512DQ-FCP-NEXT:    vpermt2d %ymm1, %ymm10, %ymm31024; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm3, %zmm11025; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]1026; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]1027; AVX512DQ-FCP-NEXT:    vpermt2d %ymm3, %ymm10, %ymm41028; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]1029; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]1030; AVX512DQ-FCP-NEXT:    vpermt2d %ymm0, %ymm10, %ymm21031; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm01032; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, (%r8)1033; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 64(%r8)1034; AVX512DQ-FCP-NEXT:    vzeroupper1035; AVX512DQ-FCP-NEXT:    retq1036;1037; AVX512BW-LABEL: store_i16_stride4_vf16:1038; AVX512BW:       # %bb.0:1039; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm01040; AVX512BW-NEXT:    vmovdqa (%rdx), %ymm11041; AVX512BW-NEXT:    vinserti64x4 $1, (%rsi), %zmm0, %zmm01042; AVX512BW-NEXT:    vinserti64x4 $1, (%rcx), %zmm1, %zmm11043; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,16,32,48,1,17,33,49,2,18,34,50,3,19,35,51,4,20,36,52,5,21,37,53,6,22,38,54,7,23,39,55]1044; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm21045; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [8,24,40,56,9,25,41,57,10,26,42,58,11,27,43,59,12,28,44,60,13,29,45,61,14,30,46,62,15,31,47,63]1046; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm31047; AVX512BW-NEXT:    vmovdqa64 %zmm3, 64(%r8)1048; AVX512BW-NEXT:    vmovdqa64 %zmm2, (%r8)1049; AVX512BW-NEXT:    vzeroupper1050; AVX512BW-NEXT:    retq1051;1052; AVX512BW-FCP-LABEL: store_i16_stride4_vf16:1053; AVX512BW-FCP:       # %bb.0:1054; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %ymm01055; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %ymm11056; AVX512BW-FCP-NEXT:    vinserti64x4 $1, (%rsi), %zmm0, %zmm01057; AVX512BW-FCP-NEXT:    vinserti64x4 $1, (%rcx), %zmm1, %zmm11058; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,16,32,48,1,17,33,49,2,18,34,50,3,19,35,51,4,20,36,52,5,21,37,53,6,22,38,54,7,23,39,55]1059; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm21060; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [8,24,40,56,9,25,41,57,10,26,42,58,11,27,43,59,12,28,44,60,13,29,45,61,14,30,46,62,15,31,47,63]1061; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm31062; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, 64(%r8)1063; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm2, (%r8)1064; AVX512BW-FCP-NEXT:    vzeroupper1065; AVX512BW-FCP-NEXT:    retq1066;1067; AVX512DQ-BW-LABEL: store_i16_stride4_vf16:1068; AVX512DQ-BW:       # %bb.0:1069; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %ymm01070; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %ymm11071; AVX512DQ-BW-NEXT:    vinserti64x4 $1, (%rsi), %zmm0, %zmm01072; AVX512DQ-BW-NEXT:    vinserti64x4 $1, (%rcx), %zmm1, %zmm11073; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,16,32,48,1,17,33,49,2,18,34,50,3,19,35,51,4,20,36,52,5,21,37,53,6,22,38,54,7,23,39,55]1074; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm21075; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [8,24,40,56,9,25,41,57,10,26,42,58,11,27,43,59,12,28,44,60,13,29,45,61,14,30,46,62,15,31,47,63]1076; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm31077; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm3, 64(%r8)1078; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm2, (%r8)1079; AVX512DQ-BW-NEXT:    vzeroupper1080; AVX512DQ-BW-NEXT:    retq1081;1082; AVX512DQ-BW-FCP-LABEL: store_i16_stride4_vf16:1083; AVX512DQ-BW-FCP:       # %bb.0:1084; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %ymm01085; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %ymm11086; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, (%rsi), %zmm0, %zmm01087; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, (%rcx), %zmm1, %zmm11088; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,16,32,48,1,17,33,49,2,18,34,50,3,19,35,51,4,20,36,52,5,21,37,53,6,22,38,54,7,23,39,55]1089; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm21090; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [8,24,40,56,9,25,41,57,10,26,42,58,11,27,43,59,12,28,44,60,13,29,45,61,14,30,46,62,15,31,47,63]1091; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm31092; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm3, 64(%r8)1093; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm2, (%r8)1094; AVX512DQ-BW-FCP-NEXT:    vzeroupper1095; AVX512DQ-BW-FCP-NEXT:    retq1096  %in.vec0 = load <16 x i16>, ptr %in.vecptr0, align 641097  %in.vec1 = load <16 x i16>, ptr %in.vecptr1, align 641098  %in.vec2 = load <16 x i16>, ptr %in.vecptr2, align 641099  %in.vec3 = load <16 x i16>, ptr %in.vecptr3, align 641100  %1 = shufflevector <16 x i16> %in.vec0, <16 x i16> %in.vec1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>1101  %2 = shufflevector <16 x i16> %in.vec2, <16 x i16> %in.vec3, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>1102  %3 = shufflevector <32 x i16> %1, <32 x i16> %2, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>1103  %interleaved.vec = shufflevector <64 x i16> %3, <64 x i16> poison, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>1104  store <64 x i16> %interleaved.vec, ptr %out.vec, align 641105  ret void1106}1107 1108define void @store_i16_stride4_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %out.vec) nounwind {1109; SSE-LABEL: store_i16_stride4_vf32:1110; SSE:       # %bb.0:1111; SSE-NEXT:    movdqa (%rdi), %xmm51112; SSE-NEXT:    movdqa 16(%rdi), %xmm111113; SSE-NEXT:    movdqa 32(%rdi), %xmm41114; SSE-NEXT:    movdqa 48(%rdi), %xmm21115; SSE-NEXT:    movdqa (%rsi), %xmm01116; SSE-NEXT:    movdqa 16(%rsi), %xmm31117; SSE-NEXT:    movdqa 32(%rsi), %xmm91118; SSE-NEXT:    movdqa (%rdx), %xmm71119; SSE-NEXT:    movdqa 16(%rdx), %xmm131120; SSE-NEXT:    movdqa 32(%rdx), %xmm101121; SSE-NEXT:    movdqa (%rcx), %xmm81122; SSE-NEXT:    movdqa 16(%rcx), %xmm141123; SSE-NEXT:    movdqa 32(%rcx), %xmm121124; SSE-NEXT:    movdqa %xmm7, %xmm151125; SSE-NEXT:    punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3]1126; SSE-NEXT:    movdqa %xmm5, %xmm61127; SSE-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]1128; SSE-NEXT:    movdqa %xmm6, %xmm11129; SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm15[2],xmm1[3],xmm15[3]1130; SSE-NEXT:    movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1131; SSE-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm15[0],xmm6[1],xmm15[1]1132; SSE-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7]1133; SSE-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]1134; SSE-NEXT:    movdqa %xmm5, %xmm01135; SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm7[2],xmm0[3],xmm7[3]1136; SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1137; SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]1138; SSE-NEXT:    movdqa %xmm13, %xmm151139; SSE-NEXT:    punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]1140; SSE-NEXT:    movdqa %xmm11, %xmm71141; SSE-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]1142; SSE-NEXT:    movdqa %xmm7, %xmm01143; SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm15[2],xmm0[3],xmm15[3]1144; SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1145; SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm15[0],xmm7[1],xmm15[1]1146; SSE-NEXT:    punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm14[4],xmm13[5],xmm14[5],xmm13[6],xmm14[6],xmm13[7],xmm14[7]1147; SSE-NEXT:    punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm3[4],xmm11[5],xmm3[5],xmm11[6],xmm3[6],xmm11[7],xmm3[7]1148; SSE-NEXT:    movdqa %xmm11, %xmm81149; SSE-NEXT:    punpckhdq {{.*#+}} xmm8 = xmm8[2],xmm13[2],xmm8[3],xmm13[3]1150; SSE-NEXT:    punpckldq {{.*#+}} xmm11 = xmm11[0],xmm13[0],xmm11[1],xmm13[1]1151; SSE-NEXT:    movdqa %xmm10, %xmm151152; SSE-NEXT:    punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm12[0],xmm15[1],xmm12[1],xmm15[2],xmm12[2],xmm15[3],xmm12[3]1153; SSE-NEXT:    movdqa %xmm4, %xmm131154; SSE-NEXT:    punpcklwd {{.*#+}} xmm13 = xmm13[0],xmm9[0],xmm13[1],xmm9[1],xmm13[2],xmm9[2],xmm13[3],xmm9[3]1155; SSE-NEXT:    movdqa %xmm13, %xmm141156; SSE-NEXT:    punpckhdq {{.*#+}} xmm14 = xmm14[2],xmm15[2],xmm14[3],xmm15[3]1157; SSE-NEXT:    punpckldq {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1]1158; SSE-NEXT:    movdqa 48(%rdx), %xmm151159; SSE-NEXT:    punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm12[4],xmm10[5],xmm12[5],xmm10[6],xmm12[6],xmm10[7],xmm12[7]1160; SSE-NEXT:    movdqa 48(%rcx), %xmm121161; SSE-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm9[4],xmm4[5],xmm9[5],xmm4[6],xmm9[6],xmm4[7],xmm9[7]1162; SSE-NEXT:    movdqa %xmm4, %xmm91163; SSE-NEXT:    punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm10[2],xmm9[3],xmm10[3]1164; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm10[0],xmm4[1],xmm10[1]1165; SSE-NEXT:    movdqa %xmm15, %xmm101166; SSE-NEXT:    punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1],xmm10[2],xmm12[2],xmm10[3],xmm12[3]1167; SSE-NEXT:    movdqa 48(%rsi), %xmm11168; SSE-NEXT:    movdqa %xmm2, %xmm31169; SSE-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]1170; SSE-NEXT:    movdqa %xmm3, %xmm01171; SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm10[2],xmm0[3],xmm10[3]1172; SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1]1173; SSE-NEXT:    punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm12[4],xmm15[5],xmm12[5],xmm15[6],xmm12[6],xmm15[7],xmm12[7]1174; SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]1175; SSE-NEXT:    movdqa %xmm2, %xmm11176; SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm15[2],xmm1[3],xmm15[3]1177; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm15[0],xmm2[1],xmm15[1]1178; SSE-NEXT:    movdqa %xmm2, 224(%r8)1179; SSE-NEXT:    movdqa %xmm1, 240(%r8)1180; SSE-NEXT:    movdqa %xmm3, 192(%r8)1181; SSE-NEXT:    movdqa %xmm0, 208(%r8)1182; SSE-NEXT:    movdqa %xmm4, 160(%r8)1183; SSE-NEXT:    movdqa %xmm9, 176(%r8)1184; SSE-NEXT:    movdqa %xmm13, 128(%r8)1185; SSE-NEXT:    movdqa %xmm14, 144(%r8)1186; SSE-NEXT:    movdqa %xmm11, 96(%r8)1187; SSE-NEXT:    movdqa %xmm8, 112(%r8)1188; SSE-NEXT:    movdqa %xmm7, 64(%r8)1189; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1190; SSE-NEXT:    movaps %xmm0, 80(%r8)1191; SSE-NEXT:    movdqa %xmm5, 32(%r8)1192; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1193; SSE-NEXT:    movaps %xmm0, 48(%r8)1194; SSE-NEXT:    movdqa %xmm6, (%r8)1195; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1196; SSE-NEXT:    movaps %xmm0, 16(%r8)1197; SSE-NEXT:    retq1198;1199; AVX-LABEL: store_i16_stride4_vf32:1200; AVX:       # %bb.0:1201; AVX-NEXT:    vmovdqa (%rcx), %xmm31202; AVX-NEXT:    vmovdqa 16(%rcx), %xmm01203; AVX-NEXT:    vmovdqa 32(%rcx), %xmm41204; AVX-NEXT:    vmovdqa 48(%rcx), %xmm71205; AVX-NEXT:    vmovdqa (%rdx), %xmm61206; AVX-NEXT:    vmovdqa 16(%rdx), %xmm11207; AVX-NEXT:    vmovdqa 32(%rdx), %xmm51208; AVX-NEXT:    vmovdqa 48(%rdx), %xmm81209; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]1210; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm2[0,0,1,1]1211; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]1212; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm9, %ymm21213; AVX-NEXT:    vmovdqa (%rsi), %xmm91214; AVX-NEXT:    vmovdqa (%rdi), %xmm101215; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]1216; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm12 = xmm11[0],zero,xmm11[1],zero1217; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,2,3,3]1218; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm12, %ymm111219; AVX-NEXT:    vmovdqa 48(%rsi), %xmm121220; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm11[0],ymm2[1],ymm11[2],ymm2[3],ymm11[4],ymm2[5],ymm11[6],ymm2[7]1221; AVX-NEXT:    vmovdqa 48(%rdi), %xmm111222; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]1223; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[0,0,1,1]1224; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,2,3,3]1225; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm6, %ymm31226; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]1227; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm9 = xmm6[0],zero,xmm6[1],zero1228; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]1229; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm9, %ymm61230; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7]1231; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]1232; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm6[0,0,1,1]1233; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]1234; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm9, %ymm61235; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]1236; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm10 = xmm9[0],zero,xmm9[1],zero1237; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]1238; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm10, %ymm91239; AVX-NEXT:    vmovdqa 32(%rsi), %xmm101240; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm9[0],ymm6[1],ymm9[2],ymm6[3],ymm9[4],ymm6[5],ymm9[6],ymm6[7]1241; AVX-NEXT:    vmovdqa 32(%rdi), %xmm91242; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]1243; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm7[0,0,1,1]1244; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]1245; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm8, %ymm71246; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]1247; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm11 = xmm8[0],zero,xmm8[1],zero1248; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]1249; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm81250; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4],ymm7[5],ymm8[6],ymm7[7]1251; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]1252; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm8[0,0,1,1]1253; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]1254; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm81255; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]1256; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm12 = xmm11[0],zero,xmm11[1],zero1257; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,2,3,3]1258; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm12, %ymm111259; AVX-NEXT:    vmovdqa 16(%rsi), %xmm121260; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm11[0],ymm8[1],ymm11[2],ymm8[3],ymm11[4],ymm8[5],ymm11[6],ymm8[7]1261; AVX-NEXT:    vmovdqa 16(%rdi), %xmm111262; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]1263; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[0,0,1,1]1264; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]1265; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm41266; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm9[4],xmm10[4],xmm9[5],xmm10[5],xmm9[6],xmm10[6],xmm9[7],xmm10[7]1267; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm9 = xmm5[0],zero,xmm5[1],zero1268; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]1269; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm9, %ymm51270; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7]1271; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]1272; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm5[0,0,1,1]1273; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]1274; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm9, %ymm51275; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]1276; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm10 = xmm9[0],zero,xmm9[1],zero1277; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]1278; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm10, %ymm91279; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7]1280; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]1281; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]1282; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]1283; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm01284; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]1285; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm9 = xmm1[0],zero,xmm1[1],zero1286; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]1287; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm9, %ymm11288; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]1289; AVX-NEXT:    vmovaps %ymm0, 96(%r8)1290; AVX-NEXT:    vmovaps %ymm5, 64(%r8)1291; AVX-NEXT:    vmovaps %ymm4, 160(%r8)1292; AVX-NEXT:    vmovaps %ymm8, 128(%r8)1293; AVX-NEXT:    vmovaps %ymm7, 224(%r8)1294; AVX-NEXT:    vmovaps %ymm6, 192(%r8)1295; AVX-NEXT:    vmovaps %ymm3, (%r8)1296; AVX-NEXT:    vmovaps %ymm2, 32(%r8)1297; AVX-NEXT:    vzeroupper1298; AVX-NEXT:    retq1299;1300; AVX2-LABEL: store_i16_stride4_vf32:1301; AVX2:       # %bb.0:1302; AVX2-NEXT:    vmovdqa (%rcx), %xmm31303; AVX2-NEXT:    vmovdqa 16(%rcx), %xmm01304; AVX2-NEXT:    vmovdqa 32(%rcx), %xmm41305; AVX2-NEXT:    vmovdqa 48(%rcx), %xmm71306; AVX2-NEXT:    vmovdqa (%rdx), %xmm61307; AVX2-NEXT:    vmovdqa 16(%rdx), %xmm11308; AVX2-NEXT:    vmovdqa 32(%rdx), %xmm51309; AVX2-NEXT:    vmovdqa 48(%rdx), %xmm81310; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]1311; AVX2-NEXT:    vpshufd {{.*#+}} xmm9 = xmm2[0,0,1,1]1312; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]1313; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm9, %ymm21314; AVX2-NEXT:    vmovdqa (%rsi), %xmm91315; AVX2-NEXT:    vmovdqa (%rdi), %xmm101316; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]1317; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm12 = xmm11[0],zero,xmm11[1],zero1318; AVX2-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,2,3,3]1319; AVX2-NEXT:    vinserti128 $1, %xmm11, %ymm12, %ymm111320; AVX2-NEXT:    vmovdqa 48(%rsi), %xmm121321; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm11[0],ymm2[1],ymm11[2],ymm2[3],ymm11[4],ymm2[5],ymm11[6],ymm2[7]1322; AVX2-NEXT:    vmovdqa 48(%rdi), %xmm111323; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]1324; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[0,0,1,1]1325; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,2,3,3]1326; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm6, %ymm31327; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]1328; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm9 = xmm6[0],zero,xmm6[1],zero1329; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]1330; AVX2-NEXT:    vinserti128 $1, %xmm6, %ymm9, %ymm61331; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7]1332; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]1333; AVX2-NEXT:    vpshufd {{.*#+}} xmm9 = xmm6[0,0,1,1]1334; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]1335; AVX2-NEXT:    vinserti128 $1, %xmm6, %ymm9, %ymm61336; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]1337; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm10 = xmm9[0],zero,xmm9[1],zero1338; AVX2-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]1339; AVX2-NEXT:    vinserti128 $1, %xmm9, %ymm10, %ymm91340; AVX2-NEXT:    vmovdqa 32(%rsi), %xmm101341; AVX2-NEXT:    vpblendd {{.*#+}} ymm6 = ymm9[0],ymm6[1],ymm9[2],ymm6[3],ymm9[4],ymm6[5],ymm9[6],ymm6[7]1342; AVX2-NEXT:    vmovdqa 32(%rdi), %xmm91343; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]1344; AVX2-NEXT:    vpshufd {{.*#+}} xmm8 = xmm7[0,0,1,1]1345; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]1346; AVX2-NEXT:    vinserti128 $1, %xmm7, %ymm8, %ymm71347; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]1348; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm11 = xmm8[0],zero,xmm8[1],zero1349; AVX2-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]1350; AVX2-NEXT:    vinserti128 $1, %xmm8, %ymm11, %ymm81351; AVX2-NEXT:    vpblendd {{.*#+}} ymm7 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4],ymm7[5],ymm8[6],ymm7[7]1352; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]1353; AVX2-NEXT:    vpshufd {{.*#+}} xmm11 = xmm8[0,0,1,1]1354; AVX2-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]1355; AVX2-NEXT:    vinserti128 $1, %xmm8, %ymm11, %ymm81356; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]1357; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm12 = xmm11[0],zero,xmm11[1],zero1358; AVX2-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,2,3,3]1359; AVX2-NEXT:    vinserti128 $1, %xmm11, %ymm12, %ymm111360; AVX2-NEXT:    vmovdqa 16(%rsi), %xmm121361; AVX2-NEXT:    vpblendd {{.*#+}} ymm8 = ymm11[0],ymm8[1],ymm11[2],ymm8[3],ymm11[4],ymm8[5],ymm11[6],ymm8[7]1362; AVX2-NEXT:    vmovdqa 16(%rdi), %xmm111363; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]1364; AVX2-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[0,0,1,1]1365; AVX2-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]1366; AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm41367; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm9[4],xmm10[4],xmm9[5],xmm10[5],xmm9[6],xmm10[6],xmm9[7],xmm10[7]1368; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm9 = xmm5[0],zero,xmm5[1],zero1369; AVX2-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]1370; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm9, %ymm51371; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7]1372; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]1373; AVX2-NEXT:    vpshufd {{.*#+}} xmm9 = xmm5[0,0,1,1]1374; AVX2-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]1375; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm9, %ymm51376; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]1377; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm10 = xmm9[0],zero,xmm9[1],zero1378; AVX2-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]1379; AVX2-NEXT:    vinserti128 $1, %xmm9, %ymm10, %ymm91380; AVX2-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7]1381; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]1382; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]1383; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]1384; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm01385; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]1386; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm9 = xmm1[0],zero,xmm1[1],zero1387; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]1388; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm9, %ymm11389; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]1390; AVX2-NEXT:    vmovdqa %ymm0, 96(%r8)1391; AVX2-NEXT:    vmovdqa %ymm5, 64(%r8)1392; AVX2-NEXT:    vmovdqa %ymm4, 160(%r8)1393; AVX2-NEXT:    vmovdqa %ymm8, 128(%r8)1394; AVX2-NEXT:    vmovdqa %ymm7, 224(%r8)1395; AVX2-NEXT:    vmovdqa %ymm6, 192(%r8)1396; AVX2-NEXT:    vmovdqa %ymm3, (%r8)1397; AVX2-NEXT:    vmovdqa %ymm2, 32(%r8)1398; AVX2-NEXT:    vzeroupper1399; AVX2-NEXT:    retq1400;1401; AVX2-FP-LABEL: store_i16_stride4_vf32:1402; AVX2-FP:       # %bb.0:1403; AVX2-FP-NEXT:    vmovdqa (%rcx), %xmm31404; AVX2-FP-NEXT:    vmovdqa 16(%rcx), %xmm01405; AVX2-FP-NEXT:    vmovdqa 32(%rcx), %xmm41406; AVX2-FP-NEXT:    vmovdqa 48(%rcx), %xmm71407; AVX2-FP-NEXT:    vmovdqa (%rdx), %xmm61408; AVX2-FP-NEXT:    vmovdqa 16(%rdx), %xmm11409; AVX2-FP-NEXT:    vmovdqa 32(%rdx), %xmm51410; AVX2-FP-NEXT:    vmovdqa 48(%rdx), %xmm81411; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]1412; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm2[0,0,1,1]1413; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]1414; AVX2-FP-NEXT:    vinserti128 $1, %xmm2, %ymm9, %ymm21415; AVX2-FP-NEXT:    vmovdqa (%rsi), %xmm91416; AVX2-FP-NEXT:    vmovdqa (%rdi), %xmm101417; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]1418; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm12 = xmm11[0],zero,xmm11[1],zero1419; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,2,3,3]1420; AVX2-FP-NEXT:    vinserti128 $1, %xmm11, %ymm12, %ymm111421; AVX2-FP-NEXT:    vmovdqa 48(%rsi), %xmm121422; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm11[0],ymm2[1],ymm11[2],ymm2[3],ymm11[4],ymm2[5],ymm11[6],ymm2[7]1423; AVX2-FP-NEXT:    vmovdqa 48(%rdi), %xmm111424; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]1425; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[0,0,1,1]1426; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,2,3,3]1427; AVX2-FP-NEXT:    vinserti128 $1, %xmm3, %ymm6, %ymm31428; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]1429; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm9 = xmm6[0],zero,xmm6[1],zero1430; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]1431; AVX2-FP-NEXT:    vinserti128 $1, %xmm6, %ymm9, %ymm61432; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0],ymm3[1],ymm6[2],ymm3[3],ymm6[4],ymm3[5],ymm6[6],ymm3[7]1433; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]1434; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm6[0,0,1,1]1435; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]1436; AVX2-FP-NEXT:    vinserti128 $1, %xmm6, %ymm9, %ymm61437; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]1438; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm10 = xmm9[0],zero,xmm9[1],zero1439; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]1440; AVX2-FP-NEXT:    vinserti128 $1, %xmm9, %ymm10, %ymm91441; AVX2-FP-NEXT:    vmovdqa 32(%rsi), %xmm101442; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm9[0],ymm6[1],ymm9[2],ymm6[3],ymm9[4],ymm6[5],ymm9[6],ymm6[7]1443; AVX2-FP-NEXT:    vmovdqa 32(%rdi), %xmm91444; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]1445; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm7[0,0,1,1]1446; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]1447; AVX2-FP-NEXT:    vinserti128 $1, %xmm7, %ymm8, %ymm71448; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]1449; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm11 = xmm8[0],zero,xmm8[1],zero1450; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]1451; AVX2-FP-NEXT:    vinserti128 $1, %xmm8, %ymm11, %ymm81452; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4],ymm7[5],ymm8[6],ymm7[7]1453; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]1454; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm11 = xmm8[0,0,1,1]1455; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]1456; AVX2-FP-NEXT:    vinserti128 $1, %xmm8, %ymm11, %ymm81457; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]1458; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm12 = xmm11[0],zero,xmm11[1],zero1459; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,2,3,3]1460; AVX2-FP-NEXT:    vinserti128 $1, %xmm11, %ymm12, %ymm111461; AVX2-FP-NEXT:    vmovdqa 16(%rsi), %xmm121462; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm8 = ymm11[0],ymm8[1],ymm11[2],ymm8[3],ymm11[4],ymm8[5],ymm11[6],ymm8[7]1463; AVX2-FP-NEXT:    vmovdqa 16(%rdi), %xmm111464; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]1465; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[0,0,1,1]1466; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]1467; AVX2-FP-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm41468; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm9[4],xmm10[4],xmm9[5],xmm10[5],xmm9[6],xmm10[6],xmm9[7],xmm10[7]1469; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm9 = xmm5[0],zero,xmm5[1],zero1470; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]1471; AVX2-FP-NEXT:    vinserti128 $1, %xmm5, %ymm9, %ymm51472; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4],ymm4[5],ymm5[6],ymm4[7]1473; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]1474; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm5[0,0,1,1]1475; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]1476; AVX2-FP-NEXT:    vinserti128 $1, %xmm5, %ymm9, %ymm51477; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]1478; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm10 = xmm9[0],zero,xmm9[1],zero1479; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]1480; AVX2-FP-NEXT:    vinserti128 $1, %xmm9, %ymm10, %ymm91481; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0],ymm5[1],ymm9[2],ymm5[3],ymm9[4],ymm5[5],ymm9[6],ymm5[7]1482; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]1483; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]1484; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]1485; AVX2-FP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm01486; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]1487; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm9 = xmm1[0],zero,xmm1[1],zero1488; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]1489; AVX2-FP-NEXT:    vinserti128 $1, %xmm1, %ymm9, %ymm11490; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]1491; AVX2-FP-NEXT:    vmovdqa %ymm0, 96(%r8)1492; AVX2-FP-NEXT:    vmovdqa %ymm5, 64(%r8)1493; AVX2-FP-NEXT:    vmovdqa %ymm4, 160(%r8)1494; AVX2-FP-NEXT:    vmovdqa %ymm8, 128(%r8)1495; AVX2-FP-NEXT:    vmovdqa %ymm7, 224(%r8)1496; AVX2-FP-NEXT:    vmovdqa %ymm6, 192(%r8)1497; AVX2-FP-NEXT:    vmovdqa %ymm3, (%r8)1498; AVX2-FP-NEXT:    vmovdqa %ymm2, 32(%r8)1499; AVX2-FP-NEXT:    vzeroupper1500; AVX2-FP-NEXT:    retq1501;1502; AVX2-FCP-LABEL: store_i16_stride4_vf32:1503; AVX2-FCP:       # %bb.0:1504; AVX2-FCP-NEXT:    vmovdqa (%rsi), %xmm51505; AVX2-FCP-NEXT:    vmovdqa 16(%rsi), %xmm01506; AVX2-FCP-NEXT:    vmovdqa 32(%rsi), %xmm41507; AVX2-FCP-NEXT:    vmovdqa 48(%rsi), %xmm81508; AVX2-FCP-NEXT:    vmovdqa (%rdi), %xmm71509; AVX2-FCP-NEXT:    vmovdqa 16(%rdi), %xmm11510; AVX2-FCP-NEXT:    vmovdqa 32(%rdi), %xmm61511; AVX2-FCP-NEXT:    vmovdqa 48(%rdi), %xmm91512; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]1513; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero1514; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]1515; AVX2-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm21516; AVX2-FCP-NEXT:    vmovdqa (%rcx), %xmm101517; AVX2-FCP-NEXT:    vmovdqa (%rdx), %xmm111518; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm12 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]1519; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [0,0,1,1,2,2,3,3]1520; AVX2-FCP-NEXT:    vpermd %ymm12, %ymm3, %ymm121521; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm12[1],ymm2[2],ymm12[3],ymm2[4],ymm12[5],ymm2[6],ymm12[7]1522; AVX2-FCP-NEXT:    vmovdqa 48(%rcx), %xmm121523; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]1524; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero1525; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]1526; AVX2-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm7, %ymm51527; AVX2-FCP-NEXT:    vmovdqa 48(%rdx), %xmm131528; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]1529; AVX2-FCP-NEXT:    vpermd %ymm7, %ymm3, %ymm71530; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0],ymm7[1],ymm5[2],ymm7[3],ymm5[4],ymm7[5],ymm5[6],ymm7[7]1531; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]1532; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm10 = xmm7[0],zero,xmm7[1],zero1533; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]1534; AVX2-FCP-NEXT:    vinserti128 $1, %xmm7, %ymm10, %ymm71535; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]1536; AVX2-FCP-NEXT:    vpermd %ymm10, %ymm3, %ymm101537; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0],ymm10[1],ymm7[2],ymm10[3],ymm7[4],ymm10[5],ymm7[6],ymm10[7]1538; AVX2-FCP-NEXT:    vmovdqa 32(%rcx), %xmm101539; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]1540; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm9 = xmm8[0],zero,xmm8[1],zero1541; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]1542; AVX2-FCP-NEXT:    vinserti128 $1, %xmm8, %ymm9, %ymm81543; AVX2-FCP-NEXT:    vmovdqa 32(%rdx), %xmm111544; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7]1545; AVX2-FCP-NEXT:    vpermd %ymm9, %ymm3, %ymm91546; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2],ymm9[3],ymm8[4],ymm9[5],ymm8[6],ymm9[7]1547; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]1548; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm12 = xmm9[0],zero,xmm9[1],zero1549; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]1550; AVX2-FCP-NEXT:    vinserti128 $1, %xmm9, %ymm12, %ymm91551; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]1552; AVX2-FCP-NEXT:    vpermd %ymm12, %ymm3, %ymm121553; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0],ymm12[1],ymm9[2],ymm12[3],ymm9[4],ymm12[5],ymm9[6],ymm12[7]1554; AVX2-FCP-NEXT:    vmovdqa 16(%rcx), %xmm121555; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]1556; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm4[0],zero,xmm4[1],zero1557; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]1558; AVX2-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm6, %ymm41559; AVX2-FCP-NEXT:    vmovdqa 16(%rdx), %xmm61560; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]1561; AVX2-FCP-NEXT:    vpermd %ymm10, %ymm3, %ymm101562; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0],ymm10[1],ymm4[2],ymm10[3],ymm4[4],ymm10[5],ymm4[6],ymm10[7]1563; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]1564; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm11 = xmm10[0],zero,xmm10[1],zero1565; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm10 = xmm10[2,2,3,3]1566; AVX2-FCP-NEXT:    vinserti128 $1, %xmm10, %ymm11, %ymm101567; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm6[0],xmm12[0],xmm6[1],xmm12[1],xmm6[2],xmm12[2],xmm6[3],xmm12[3]1568; AVX2-FCP-NEXT:    vpermd %ymm11, %ymm3, %ymm111569; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4],ymm11[5],ymm10[6],ymm11[7]1570; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]1571; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero1572; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]1573; AVX2-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm01574; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm6[4],xmm12[4],xmm6[5],xmm12[5],xmm6[6],xmm12[6],xmm6[7],xmm12[7]1575; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm11576; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]1577; AVX2-FCP-NEXT:    vmovdqa %ymm0, 96(%r8)1578; AVX2-FCP-NEXT:    vmovdqa %ymm10, 64(%r8)1579; AVX2-FCP-NEXT:    vmovdqa %ymm4, 160(%r8)1580; AVX2-FCP-NEXT:    vmovdqa %ymm9, 128(%r8)1581; AVX2-FCP-NEXT:    vmovdqa %ymm8, 224(%r8)1582; AVX2-FCP-NEXT:    vmovdqa %ymm7, 192(%r8)1583; AVX2-FCP-NEXT:    vmovdqa %ymm5, (%r8)1584; AVX2-FCP-NEXT:    vmovdqa %ymm2, 32(%r8)1585; AVX2-FCP-NEXT:    vzeroupper1586; AVX2-FCP-NEXT:    retq1587;1588; AVX512-LABEL: store_i16_stride4_vf32:1589; AVX512:       # %bb.0:1590; AVX512-NEXT:    vmovdqa (%rsi), %xmm01591; AVX512-NEXT:    vmovdqa 16(%rsi), %xmm11592; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm101593; AVX512-NEXT:    vmovdqa 48(%rsi), %xmm61594; AVX512-NEXT:    vmovdqa (%rdi), %xmm21595; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm31596; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm111597; AVX512-NEXT:    vmovdqa 48(%rdi), %xmm71598; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]1599; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero1600; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]1601; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm41602; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]1603; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero1604; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]1605; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm3, %ymm11606; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm11607; AVX512-NEXT:    vmovdqa (%rcx), %xmm31608; AVX512-NEXT:    vmovdqa 16(%rcx), %xmm51609; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm121610; AVX512-NEXT:    vmovdqa 48(%rcx), %xmm81611; AVX512-NEXT:    vmovdqa (%rdx), %xmm41612; AVX512-NEXT:    vmovdqa 16(%rdx), %xmm131613; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm141614; AVX512-NEXT:    vmovdqa 48(%rdx), %xmm91615; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm13[4],xmm5[4],xmm13[5],xmm5[5],xmm13[6],xmm5[6],xmm13[7],xmm5[7]1616; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]1617; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm5 = [0,0,1,1,2,2,3,3,16,16,17,17,18,18,19,19]1618; AVX512-NEXT:    vpermt2d %zmm15, %zmm5, %zmm131619; AVX512-NEXT:    movw $-21846, %ax # imm = 0xAAAA1620; AVX512-NEXT:    kmovw %eax, %k11621; AVX512-NEXT:    vmovdqa32 %zmm13, %zmm1 {%k1}1622; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm13 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]1623; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm15 = xmm13[0],zero,xmm13[1],zero1624; AVX512-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[2,2,3,3]1625; AVX512-NEXT:    vinserti128 $1, %xmm13, %ymm15, %ymm131626; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]1627; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm11 = xmm10[0],zero,xmm10[1],zero1628; AVX512-NEXT:    vpshufd {{.*#+}} xmm10 = xmm10[2,2,3,3]1629; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm11, %ymm101630; AVX512-NEXT:    vinserti64x4 $1, %ymm13, %zmm10, %zmm101631; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm14[4],xmm12[4],xmm14[5],xmm12[5],xmm14[6],xmm12[6],xmm14[7],xmm12[7]1632; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm14[0],xmm12[0],xmm14[1],xmm12[1],xmm14[2],xmm12[2],xmm14[3],xmm12[3]1633; AVX512-NEXT:    vpermt2d %zmm11, %zmm5, %zmm121634; AVX512-NEXT:    vmovdqa32 %zmm12, %zmm10 {%k1}1635; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]1636; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm12 = xmm11[0],zero,xmm11[1],zero1637; AVX512-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,2,3,3]1638; AVX512-NEXT:    vinserti128 $1, %xmm11, %ymm12, %ymm111639; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]1640; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero1641; AVX512-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]1642; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm7, %ymm61643; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm6, %zmm61644; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]1645; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]1646; AVX512-NEXT:    vpermt2d %zmm7, %zmm5, %zmm81647; AVX512-NEXT:    vmovdqa32 %zmm8, %zmm6 {%k1}1648; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]1649; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm7[0],zero,xmm7[1],zero1650; AVX512-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]1651; AVX512-NEXT:    vinserti128 $1, %xmm7, %ymm8, %ymm71652; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]1653; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero1654; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]1655; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm01656; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm01657; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]1658; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]1659; AVX512-NEXT:    vpermt2d %zmm2, %zmm5, %zmm31660; AVX512-NEXT:    vmovdqa32 %zmm3, %zmm0 {%k1}1661; AVX512-NEXT:    vmovdqa64 %zmm0, (%r8)1662; AVX512-NEXT:    vmovdqa64 %zmm6, 192(%r8)1663; AVX512-NEXT:    vmovdqa64 %zmm10, 128(%r8)1664; AVX512-NEXT:    vmovdqa64 %zmm1, 64(%r8)1665; AVX512-NEXT:    vzeroupper1666; AVX512-NEXT:    retq1667;1668; AVX512-FCP-LABEL: store_i16_stride4_vf32:1669; AVX512-FCP:       # %bb.0:1670; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm01671; AVX512-FCP-NEXT:    vmovdqa 16(%rsi), %xmm11672; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm101673; AVX512-FCP-NEXT:    vmovdqa 48(%rsi), %xmm61674; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm21675; AVX512-FCP-NEXT:    vmovdqa 16(%rdi), %xmm31676; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm111677; AVX512-FCP-NEXT:    vmovdqa 48(%rdi), %xmm71678; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]1679; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero1680; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]1681; AVX512-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm41682; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]1683; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero1684; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]1685; AVX512-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm3, %ymm11686; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm11687; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm31688; AVX512-FCP-NEXT:    vmovdqa 16(%rcx), %xmm51689; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm121690; AVX512-FCP-NEXT:    vmovdqa 48(%rcx), %xmm81691; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm41692; AVX512-FCP-NEXT:    vmovdqa 16(%rdx), %xmm131693; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm141694; AVX512-FCP-NEXT:    vmovdqa 48(%rdx), %xmm91695; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm13[4],xmm5[4],xmm13[5],xmm5[5],xmm13[6],xmm5[6],xmm13[7],xmm5[7]1696; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]1697; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm5 = [0,0,1,1,2,2,3,3,16,16,17,17,18,18,19,19]1698; AVX512-FCP-NEXT:    vpermt2d %zmm15, %zmm5, %zmm131699; AVX512-FCP-NEXT:    movw $-21846, %ax # imm = 0xAAAA1700; AVX512-FCP-NEXT:    kmovw %eax, %k11701; AVX512-FCP-NEXT:    vmovdqa32 %zmm13, %zmm1 {%k1}1702; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm13 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]1703; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm15 = xmm13[0],zero,xmm13[1],zero1704; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[2,2,3,3]1705; AVX512-FCP-NEXT:    vinserti128 $1, %xmm13, %ymm15, %ymm131706; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]1707; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm11 = xmm10[0],zero,xmm10[1],zero1708; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm10 = xmm10[2,2,3,3]1709; AVX512-FCP-NEXT:    vinserti128 $1, %xmm10, %ymm11, %ymm101710; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm10, %zmm101711; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm14[4],xmm12[4],xmm14[5],xmm12[5],xmm14[6],xmm12[6],xmm14[7],xmm12[7]1712; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm14[0],xmm12[0],xmm14[1],xmm12[1],xmm14[2],xmm12[2],xmm14[3],xmm12[3]1713; AVX512-FCP-NEXT:    vpermt2d %zmm11, %zmm5, %zmm121714; AVX512-FCP-NEXT:    vmovdqa32 %zmm12, %zmm10 {%k1}1715; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]1716; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm12 = xmm11[0],zero,xmm11[1],zero1717; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,2,3,3]1718; AVX512-FCP-NEXT:    vinserti128 $1, %xmm11, %ymm12, %ymm111719; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]1720; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero1721; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]1722; AVX512-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm7, %ymm61723; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm6, %zmm61724; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]1725; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]1726; AVX512-FCP-NEXT:    vpermt2d %zmm7, %zmm5, %zmm81727; AVX512-FCP-NEXT:    vmovdqa32 %zmm8, %zmm6 {%k1}1728; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]1729; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm7[0],zero,xmm7[1],zero1730; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]1731; AVX512-FCP-NEXT:    vinserti128 $1, %xmm7, %ymm8, %ymm71732; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]1733; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero1734; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]1735; AVX512-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm01736; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm01737; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]1738; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]1739; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm5, %zmm31740; AVX512-FCP-NEXT:    vmovdqa32 %zmm3, %zmm0 {%k1}1741; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, (%r8)1742; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, 192(%r8)1743; AVX512-FCP-NEXT:    vmovdqa64 %zmm10, 128(%r8)1744; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 64(%r8)1745; AVX512-FCP-NEXT:    vzeroupper1746; AVX512-FCP-NEXT:    retq1747;1748; AVX512DQ-LABEL: store_i16_stride4_vf32:1749; AVX512DQ:       # %bb.0:1750; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm01751; AVX512DQ-NEXT:    vmovdqa 16(%rsi), %xmm11752; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm101753; AVX512DQ-NEXT:    vmovdqa 48(%rsi), %xmm61754; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm21755; AVX512DQ-NEXT:    vmovdqa 16(%rdi), %xmm31756; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm111757; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm71758; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]1759; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero1760; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]1761; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm41762; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]1763; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero1764; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]1765; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm3, %ymm11766; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm11767; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm31768; AVX512DQ-NEXT:    vmovdqa 16(%rcx), %xmm51769; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm121770; AVX512DQ-NEXT:    vmovdqa 48(%rcx), %xmm81771; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm41772; AVX512DQ-NEXT:    vmovdqa 16(%rdx), %xmm131773; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm141774; AVX512DQ-NEXT:    vmovdqa 48(%rdx), %xmm91775; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm13[4],xmm5[4],xmm13[5],xmm5[5],xmm13[6],xmm5[6],xmm13[7],xmm5[7]1776; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]1777; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm5 = [0,0,1,1,2,2,3,3,16,16,17,17,18,18,19,19]1778; AVX512DQ-NEXT:    vpermt2d %zmm15, %zmm5, %zmm131779; AVX512DQ-NEXT:    movw $-21846, %ax # imm = 0xAAAA1780; AVX512DQ-NEXT:    kmovw %eax, %k11781; AVX512DQ-NEXT:    vmovdqa32 %zmm13, %zmm1 {%k1}1782; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm13 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]1783; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm15 = xmm13[0],zero,xmm13[1],zero1784; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[2,2,3,3]1785; AVX512DQ-NEXT:    vinserti128 $1, %xmm13, %ymm15, %ymm131786; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]1787; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm11 = xmm10[0],zero,xmm10[1],zero1788; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm10 = xmm10[2,2,3,3]1789; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm11, %ymm101790; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm13, %zmm10, %zmm101791; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm14[4],xmm12[4],xmm14[5],xmm12[5],xmm14[6],xmm12[6],xmm14[7],xmm12[7]1792; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm14[0],xmm12[0],xmm14[1],xmm12[1],xmm14[2],xmm12[2],xmm14[3],xmm12[3]1793; AVX512DQ-NEXT:    vpermt2d %zmm11, %zmm5, %zmm121794; AVX512DQ-NEXT:    vmovdqa32 %zmm12, %zmm10 {%k1}1795; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]1796; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm12 = xmm11[0],zero,xmm11[1],zero1797; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,2,3,3]1798; AVX512DQ-NEXT:    vinserti128 $1, %xmm11, %ymm12, %ymm111799; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]1800; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero1801; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]1802; AVX512DQ-NEXT:    vinserti128 $1, %xmm6, %ymm7, %ymm61803; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm6, %zmm61804; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]1805; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]1806; AVX512DQ-NEXT:    vpermt2d %zmm7, %zmm5, %zmm81807; AVX512DQ-NEXT:    vmovdqa32 %zmm8, %zmm6 {%k1}1808; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]1809; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm7[0],zero,xmm7[1],zero1810; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]1811; AVX512DQ-NEXT:    vinserti128 $1, %xmm7, %ymm8, %ymm71812; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]1813; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero1814; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]1815; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm01816; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm01817; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]1818; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]1819; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm5, %zmm31820; AVX512DQ-NEXT:    vmovdqa32 %zmm3, %zmm0 {%k1}1821; AVX512DQ-NEXT:    vmovdqa64 %zmm0, (%r8)1822; AVX512DQ-NEXT:    vmovdqa64 %zmm6, 192(%r8)1823; AVX512DQ-NEXT:    vmovdqa64 %zmm10, 128(%r8)1824; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 64(%r8)1825; AVX512DQ-NEXT:    vzeroupper1826; AVX512DQ-NEXT:    retq1827;1828; AVX512DQ-FCP-LABEL: store_i16_stride4_vf32:1829; AVX512DQ-FCP:       # %bb.0:1830; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm01831; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rsi), %xmm11832; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm101833; AVX512DQ-FCP-NEXT:    vmovdqa 48(%rsi), %xmm61834; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm21835; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rdi), %xmm31836; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm111837; AVX512DQ-FCP-NEXT:    vmovdqa 48(%rdi), %xmm71838; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]1839; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero1840; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]1841; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm41842; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]1843; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero1844; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]1845; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm3, %ymm11846; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm11847; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm31848; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rcx), %xmm51849; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm121850; AVX512DQ-FCP-NEXT:    vmovdqa 48(%rcx), %xmm81851; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm41852; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rdx), %xmm131853; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm141854; AVX512DQ-FCP-NEXT:    vmovdqa 48(%rdx), %xmm91855; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm13[4],xmm5[4],xmm13[5],xmm5[5],xmm13[6],xmm5[6],xmm13[7],xmm5[7]1856; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]1857; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm5 = [0,0,1,1,2,2,3,3,16,16,17,17,18,18,19,19]1858; AVX512DQ-FCP-NEXT:    vpermt2d %zmm15, %zmm5, %zmm131859; AVX512DQ-FCP-NEXT:    movw $-21846, %ax # imm = 0xAAAA1860; AVX512DQ-FCP-NEXT:    kmovw %eax, %k11861; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm13, %zmm1 {%k1}1862; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm13 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]1863; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm15 = xmm13[0],zero,xmm13[1],zero1864; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[2,2,3,3]1865; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm13, %ymm15, %ymm131866; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]1867; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm11 = xmm10[0],zero,xmm10[1],zero1868; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm10 = xmm10[2,2,3,3]1869; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm10, %ymm11, %ymm101870; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm10, %zmm101871; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm14[4],xmm12[4],xmm14[5],xmm12[5],xmm14[6],xmm12[6],xmm14[7],xmm12[7]1872; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm14[0],xmm12[0],xmm14[1],xmm12[1],xmm14[2],xmm12[2],xmm14[3],xmm12[3]1873; AVX512DQ-FCP-NEXT:    vpermt2d %zmm11, %zmm5, %zmm121874; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm12, %zmm10 {%k1}1875; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]1876; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm12 = xmm11[0],zero,xmm11[1],zero1877; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,2,3,3]1878; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm11, %ymm12, %ymm111879; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]1880; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero1881; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]1882; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm7, %ymm61883; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm6, %zmm61884; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]1885; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]1886; AVX512DQ-FCP-NEXT:    vpermt2d %zmm7, %zmm5, %zmm81887; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm8, %zmm6 {%k1}1888; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]1889; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm7[0],zero,xmm7[1],zero1890; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]1891; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm7, %ymm8, %ymm71892; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]1893; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero1894; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]1895; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm01896; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm01897; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]1898; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]1899; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm5, %zmm31900; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm3, %zmm0 {%k1}1901; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, (%r8)1902; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, 192(%r8)1903; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm10, 128(%r8)1904; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 64(%r8)1905; AVX512DQ-FCP-NEXT:    vzeroupper1906; AVX512DQ-FCP-NEXT:    retq1907;1908; AVX512BW-LABEL: store_i16_stride4_vf32:1909; AVX512BW:       # %bb.0:1910; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm01911; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm11912; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm21913; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm31914; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39]1915; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm41916; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39,0,0]1917; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm51918; AVX512BW-NEXT:    movw $-21846, %ax # imm = 0xAAAA1919; AVX512BW-NEXT:    kmovd %eax, %k11920; AVX512BW-NEXT:    vmovdqa32 %zmm4, %zmm5 {%k1}1921; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,8,40,0,0,9,41,0,0,10,42,0,0,11,43,0,0,12,44,0,0,13,45,0,0,14,46,0,0,15,47]1922; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm41923; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [8,40,0,0,9,41,0,0,10,42,0,0,11,43,0,0,12,44,0,0,13,45,0,0,14,46,0,0,15,47,0,0]1924; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm61925; AVX512BW-NEXT:    vmovdqa32 %zmm4, %zmm6 {%k1}1926; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,16,48,0,0,17,49,0,0,18,50,0,0,19,51,0,0,20,52,0,0,21,53,0,0,22,54,0,0,23,55]1927; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm41928; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [16,48,0,0,17,49,0,0,18,50,0,0,19,51,0,0,20,52,0,0,21,53,0,0,22,54,0,0,23,55,0,0]1929; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm71930; AVX512BW-NEXT:    vmovdqa32 %zmm4, %zmm7 {%k1}1931; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,24,56,0,0,25,57,0,0,26,58,0,0,27,59,0,0,28,60,0,0,29,61,0,0,30,62,0,0,31,63]1932; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm41933; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [24,56,0,0,25,57,0,0,26,58,0,0,27,59,0,0,28,60,0,0,29,61,0,0,30,62,0,0,31,63,0,0]1934; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm21935; AVX512BW-NEXT:    vmovdqa32 %zmm4, %zmm2 {%k1}1936; AVX512BW-NEXT:    vmovdqa64 %zmm2, 192(%r8)1937; AVX512BW-NEXT:    vmovdqa64 %zmm7, 128(%r8)1938; AVX512BW-NEXT:    vmovdqa64 %zmm6, 64(%r8)1939; AVX512BW-NEXT:    vmovdqa64 %zmm5, (%r8)1940; AVX512BW-NEXT:    vzeroupper1941; AVX512BW-NEXT:    retq1942;1943; AVX512BW-FCP-LABEL: store_i16_stride4_vf32:1944; AVX512BW-FCP:       # %bb.0:1945; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdi), %zmm01946; AVX512BW-FCP-NEXT:    vmovdqa64 (%rsi), %zmm11947; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdx), %zmm21948; AVX512BW-FCP-NEXT:    vmovdqa64 (%rcx), %zmm31949; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39]1950; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm41951; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39,0,0]1952; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm51953; AVX512BW-FCP-NEXT:    movw $-21846, %ax # imm = 0xAAAA1954; AVX512BW-FCP-NEXT:    kmovd %eax, %k11955; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm4, %zmm5 {%k1}1956; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,8,40,0,0,9,41,0,0,10,42,0,0,11,43,0,0,12,44,0,0,13,45,0,0,14,46,0,0,15,47]1957; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm41958; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [8,40,0,0,9,41,0,0,10,42,0,0,11,43,0,0,12,44,0,0,13,45,0,0,14,46,0,0,15,47,0,0]1959; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm61960; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm4, %zmm6 {%k1}1961; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,16,48,0,0,17,49,0,0,18,50,0,0,19,51,0,0,20,52,0,0,21,53,0,0,22,54,0,0,23,55]1962; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm41963; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [16,48,0,0,17,49,0,0,18,50,0,0,19,51,0,0,20,52,0,0,21,53,0,0,22,54,0,0,23,55,0,0]1964; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm71965; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm4, %zmm7 {%k1}1966; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,24,56,0,0,25,57,0,0,26,58,0,0,27,59,0,0,28,60,0,0,29,61,0,0,30,62,0,0,31,63]1967; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm41968; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [24,56,0,0,25,57,0,0,26,58,0,0,27,59,0,0,28,60,0,0,29,61,0,0,30,62,0,0,31,63,0,0]1969; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm21970; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm4, %zmm2 {%k1}1971; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm2, 192(%r8)1972; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm7, 128(%r8)1973; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm6, 64(%r8)1974; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm5, (%r8)1975; AVX512BW-FCP-NEXT:    vzeroupper1976; AVX512BW-FCP-NEXT:    retq1977;1978; AVX512DQ-BW-LABEL: store_i16_stride4_vf32:1979; AVX512DQ-BW:       # %bb.0:1980; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdi), %zmm01981; AVX512DQ-BW-NEXT:    vmovdqa64 (%rsi), %zmm11982; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdx), %zmm21983; AVX512DQ-BW-NEXT:    vmovdqa64 (%rcx), %zmm31984; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39]1985; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm41986; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39,0,0]1987; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm51988; AVX512DQ-BW-NEXT:    movw $-21846, %ax # imm = 0xAAAA1989; AVX512DQ-BW-NEXT:    kmovd %eax, %k11990; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm4, %zmm5 {%k1}1991; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,8,40,0,0,9,41,0,0,10,42,0,0,11,43,0,0,12,44,0,0,13,45,0,0,14,46,0,0,15,47]1992; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm41993; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [8,40,0,0,9,41,0,0,10,42,0,0,11,43,0,0,12,44,0,0,13,45,0,0,14,46,0,0,15,47,0,0]1994; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm61995; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm4, %zmm6 {%k1}1996; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,16,48,0,0,17,49,0,0,18,50,0,0,19,51,0,0,20,52,0,0,21,53,0,0,22,54,0,0,23,55]1997; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm41998; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [16,48,0,0,17,49,0,0,18,50,0,0,19,51,0,0,20,52,0,0,21,53,0,0,22,54,0,0,23,55,0,0]1999; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm72000; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm4, %zmm7 {%k1}2001; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,24,56,0,0,25,57,0,0,26,58,0,0,27,59,0,0,28,60,0,0,29,61,0,0,30,62,0,0,31,63]2002; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42003; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [24,56,0,0,25,57,0,0,26,58,0,0,27,59,0,0,28,60,0,0,29,61,0,0,30,62,0,0,31,63,0,0]2004; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm22005; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm4, %zmm2 {%k1}2006; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm2, 192(%r8)2007; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm7, 128(%r8)2008; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm6, 64(%r8)2009; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm5, (%r8)2010; AVX512DQ-BW-NEXT:    vzeroupper2011; AVX512DQ-BW-NEXT:    retq2012;2013; AVX512DQ-BW-FCP-LABEL: store_i16_stride4_vf32:2014; AVX512DQ-BW-FCP:       # %bb.0:2015; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdi), %zmm02016; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rsi), %zmm12017; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdx), %zmm22018; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rcx), %zmm32019; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39]2020; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42021; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39,0,0]2022; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm52023; AVX512DQ-BW-FCP-NEXT:    movw $-21846, %ax # imm = 0xAAAA2024; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k12025; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm4, %zmm5 {%k1}2026; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,8,40,0,0,9,41,0,0,10,42,0,0,11,43,0,0,12,44,0,0,13,45,0,0,14,46,0,0,15,47]2027; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42028; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [8,40,0,0,9,41,0,0,10,42,0,0,11,43,0,0,12,44,0,0,13,45,0,0,14,46,0,0,15,47,0,0]2029; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm62030; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm4, %zmm6 {%k1}2031; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,16,48,0,0,17,49,0,0,18,50,0,0,19,51,0,0,20,52,0,0,21,53,0,0,22,54,0,0,23,55]2032; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42033; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [16,48,0,0,17,49,0,0,18,50,0,0,19,51,0,0,20,52,0,0,21,53,0,0,22,54,0,0,23,55,0,0]2034; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm72035; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm4, %zmm7 {%k1}2036; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,24,56,0,0,25,57,0,0,26,58,0,0,27,59,0,0,28,60,0,0,29,61,0,0,30,62,0,0,31,63]2037; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42038; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [24,56,0,0,25,57,0,0,26,58,0,0,27,59,0,0,28,60,0,0,29,61,0,0,30,62,0,0,31,63,0,0]2039; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm22040; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm4, %zmm2 {%k1}2041; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm2, 192(%r8)2042; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm7, 128(%r8)2043; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm6, 64(%r8)2044; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm5, (%r8)2045; AVX512DQ-BW-FCP-NEXT:    vzeroupper2046; AVX512DQ-BW-FCP-NEXT:    retq2047  %in.vec0 = load <32 x i16>, ptr %in.vecptr0, align 642048  %in.vec1 = load <32 x i16>, ptr %in.vecptr1, align 642049  %in.vec2 = load <32 x i16>, ptr %in.vecptr2, align 642050  %in.vec3 = load <32 x i16>, ptr %in.vecptr3, align 642051  %1 = shufflevector <32 x i16> %in.vec0, <32 x i16> %in.vec1, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>2052  %2 = shufflevector <32 x i16> %in.vec2, <32 x i16> %in.vec3, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>2053  %3 = shufflevector <64 x i16> %1, <64 x i16> %2, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>2054  %interleaved.vec = shufflevector <128 x i16> %3, <128 x i16> poison, <128 x i32> <i32 0, i32 32, i32 64, i32 96, i32 1, i32 33, i32 65, i32 97, i32 2, i32 34, i32 66, i32 98, i32 3, i32 35, i32 67, i32 99, i32 4, i32 36, i32 68, i32 100, i32 5, i32 37, i32 69, i32 101, i32 6, i32 38, i32 70, i32 102, i32 7, i32 39, i32 71, i32 103, i32 8, i32 40, i32 72, i32 104, i32 9, i32 41, i32 73, i32 105, i32 10, i32 42, i32 74, i32 106, i32 11, i32 43, i32 75, i32 107, i32 12, i32 44, i32 76, i32 108, i32 13, i32 45, i32 77, i32 109, i32 14, i32 46, i32 78, i32 110, i32 15, i32 47, i32 79, i32 111, i32 16, i32 48, i32 80, i32 112, i32 17, i32 49, i32 81, i32 113, i32 18, i32 50, i32 82, i32 114, i32 19, i32 51, i32 83, i32 115, i32 20, i32 52, i32 84, i32 116, i32 21, i32 53, i32 85, i32 117, i32 22, i32 54, i32 86, i32 118, i32 23, i32 55, i32 87, i32 119, i32 24, i32 56, i32 88, i32 120, i32 25, i32 57, i32 89, i32 121, i32 26, i32 58, i32 90, i32 122, i32 27, i32 59, i32 91, i32 123, i32 28, i32 60, i32 92, i32 124, i32 29, i32 61, i32 93, i32 125, i32 30, i32 62, i32 94, i32 126, i32 31, i32 63, i32 95, i32 127>2055  store <128 x i16> %interleaved.vec, ptr %out.vec, align 642056  ret void2057}2058 2059define void @store_i16_stride4_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %out.vec) nounwind {2060; SSE-LABEL: store_i16_stride4_vf64:2061; SSE:       # %bb.0:2062; SSE-NEXT:    subq $184, %rsp2063; SSE-NEXT:    movdqa (%rdi), %xmm102064; SSE-NEXT:    movdqa 16(%rdi), %xmm112065; SSE-NEXT:    movdqa 32(%rdi), %xmm122066; SSE-NEXT:    movdqa 48(%rdi), %xmm132067; SSE-NEXT:    movdqa (%rsi), %xmm52068; SSE-NEXT:    movdqa 16(%rsi), %xmm22069; SSE-NEXT:    movdqa 32(%rsi), %xmm02070; SSE-NEXT:    movdqa (%rdx), %xmm62071; SSE-NEXT:    movdqa 16(%rdx), %xmm42072; SSE-NEXT:    movdqa 32(%rdx), %xmm12073; SSE-NEXT:    movdqa (%rcx), %xmm72074; SSE-NEXT:    movdqa 16(%rcx), %xmm82075; SSE-NEXT:    movdqa 32(%rcx), %xmm32076; SSE-NEXT:    movdqa %xmm6, %xmm92077; SSE-NEXT:    punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]2078; SSE-NEXT:    movdqa %xmm10, %xmm142079; SSE-NEXT:    punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3]2080; SSE-NEXT:    movdqa %xmm14, %xmm152081; SSE-NEXT:    punpckldq {{.*#+}} xmm15 = xmm15[0],xmm9[0],xmm15[1],xmm9[1]2082; SSE-NEXT:    movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2083; SSE-NEXT:    punpckhdq {{.*#+}} xmm14 = xmm14[2],xmm9[2],xmm14[3],xmm9[3]2084; SSE-NEXT:    movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2085; SSE-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7]2086; SSE-NEXT:    punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm5[4],xmm10[5],xmm5[5],xmm10[6],xmm5[6],xmm10[7],xmm5[7]2087; SSE-NEXT:    movdqa %xmm10, %xmm52088; SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]2089; SSE-NEXT:    movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2090; SSE-NEXT:    punpckhdq {{.*#+}} xmm10 = xmm10[2],xmm6[2],xmm10[3],xmm6[3]2091; SSE-NEXT:    movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2092; SSE-NEXT:    movdqa %xmm4, %xmm52093; SSE-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1],xmm5[2],xmm8[2],xmm5[3],xmm8[3]2094; SSE-NEXT:    movdqa %xmm11, %xmm62095; SSE-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]2096; SSE-NEXT:    movdqa %xmm6, %xmm72097; SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]2098; SSE-NEXT:    movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2099; SSE-NEXT:    punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm5[2],xmm6[3],xmm5[3]2100; SSE-NEXT:    movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2101; SSE-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm8[4],xmm4[5],xmm8[5],xmm4[6],xmm8[6],xmm4[7],xmm8[7]2102; SSE-NEXT:    punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm2[4],xmm11[5],xmm2[5],xmm11[6],xmm2[6],xmm11[7],xmm2[7]2103; SSE-NEXT:    movdqa %xmm11, %xmm22104; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]2105; SSE-NEXT:    movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2106; SSE-NEXT:    punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm4[2],xmm11[3],xmm4[3]2107; SSE-NEXT:    movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2108; SSE-NEXT:    movdqa %xmm1, %xmm22109; SSE-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]2110; SSE-NEXT:    movdqa %xmm12, %xmm42111; SSE-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]2112; SSE-NEXT:    movdqa %xmm4, %xmm52113; SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]2114; SSE-NEXT:    movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2115; SSE-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]2116; SSE-NEXT:    movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2117; SSE-NEXT:    movdqa 48(%rdx), %xmm22118; SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]2119; SSE-NEXT:    movdqa 48(%rcx), %xmm32120; SSE-NEXT:    punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm0[4],xmm12[5],xmm0[5],xmm12[6],xmm0[6],xmm12[7],xmm0[7]2121; SSE-NEXT:    movdqa %xmm12, %xmm02122; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]2123; SSE-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill2124; SSE-NEXT:    punpckhdq {{.*#+}} xmm12 = xmm12[2],xmm1[2],xmm12[3],xmm1[3]2125; SSE-NEXT:    movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2126; SSE-NEXT:    movdqa %xmm2, %xmm02127; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]2128; SSE-NEXT:    movdqa 48(%rsi), %xmm12129; SSE-NEXT:    movdqa %xmm13, %xmm42130; SSE-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]2131; SSE-NEXT:    movdqa %xmm4, %xmm52132; SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]2133; SSE-NEXT:    movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2134; SSE-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm0[2],xmm4[3],xmm0[3]2135; SSE-NEXT:    movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2136; SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]2137; SSE-NEXT:    punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm1[4],xmm13[5],xmm1[5],xmm13[6],xmm1[6],xmm13[7],xmm1[7]2138; SSE-NEXT:    movdqa %xmm13, %xmm02139; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]2140; SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2141; SSE-NEXT:    punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm2[2],xmm13[3],xmm2[3]2142; SSE-NEXT:    movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2143; SSE-NEXT:    movdqa 64(%rdx), %xmm02144; SSE-NEXT:    movdqa 64(%rcx), %xmm12145; SSE-NEXT:    movdqa %xmm0, %xmm22146; SSE-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]2147; SSE-NEXT:    movdqa 64(%rdi), %xmm132148; SSE-NEXT:    movdqa 64(%rsi), %xmm32149; SSE-NEXT:    movdqa %xmm13, %xmm142150; SSE-NEXT:    punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm3[0],xmm14[1],xmm3[1],xmm14[2],xmm3[2],xmm14[3],xmm3[3]2151; SSE-NEXT:    movdqa %xmm14, %xmm42152; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]2153; SSE-NEXT:    movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2154; SSE-NEXT:    punpckhdq {{.*#+}} xmm14 = xmm14[2],xmm2[2],xmm14[3],xmm2[3]2155; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]2156; SSE-NEXT:    punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm3[4],xmm13[5],xmm3[5],xmm13[6],xmm3[6],xmm13[7],xmm3[7]2157; SSE-NEXT:    movdqa %xmm13, %xmm12158; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]2159; SSE-NEXT:    movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2160; SSE-NEXT:    punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm0[2],xmm13[3],xmm0[3]2161; SSE-NEXT:    movdqa 80(%rdx), %xmm02162; SSE-NEXT:    movdqa 80(%rcx), %xmm12163; SSE-NEXT:    movdqa %xmm0, %xmm22164; SSE-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]2165; SSE-NEXT:    movdqa 80(%rdi), %xmm112166; SSE-NEXT:    movdqa 80(%rsi), %xmm72167; SSE-NEXT:    movdqa %xmm11, %xmm82168; SSE-NEXT:    punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]2169; SSE-NEXT:    movdqa %xmm8, %xmm32170; SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]2171; SSE-NEXT:    movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2172; SSE-NEXT:    punpckhdq {{.*#+}} xmm8 = xmm8[2],xmm2[2],xmm8[3],xmm2[3]2173; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]2174; SSE-NEXT:    punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm7[4],xmm11[5],xmm7[5],xmm11[6],xmm7[6],xmm11[7],xmm7[7]2175; SSE-NEXT:    movdqa %xmm11, %xmm152176; SSE-NEXT:    punpckldq {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]2177; SSE-NEXT:    punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm0[2],xmm11[3],xmm0[3]2178; SSE-NEXT:    movdqa 96(%rdx), %xmm12179; SSE-NEXT:    movdqa 96(%rcx), %xmm62180; SSE-NEXT:    movdqa %xmm1, %xmm02181; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3]2182; SSE-NEXT:    movdqa 96(%rdi), %xmm52183; SSE-NEXT:    movdqa 96(%rsi), %xmm42184; SSE-NEXT:    movdqa %xmm5, %xmm92185; SSE-NEXT:    punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]2186; SSE-NEXT:    movdqa %xmm9, %xmm122187; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm0[0],xmm12[1],xmm0[1]2188; SSE-NEXT:    punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm0[2],xmm9[3],xmm0[3]2189; SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]2190; SSE-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]2191; SSE-NEXT:    movdqa %xmm5, %xmm102192; SSE-NEXT:    punpckldq {{.*#+}} xmm10 = xmm10[0],xmm1[0],xmm10[1],xmm1[1]2193; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm1[2],xmm5[3],xmm1[3]2194; SSE-NEXT:    movdqa 112(%rdx), %xmm22195; SSE-NEXT:    movdqa 112(%rcx), %xmm72196; SSE-NEXT:    movdqa %xmm2, %xmm62197; SSE-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]2198; SSE-NEXT:    movdqa 112(%rdi), %xmm02199; SSE-NEXT:    movdqa 112(%rsi), %xmm42200; SSE-NEXT:    movdqa %xmm0, %xmm12201; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]2202; SSE-NEXT:    movdqa %xmm1, %xmm32203; SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]2204; SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm6[2],xmm1[3],xmm6[3]2205; SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm7[4],xmm2[5],xmm7[5],xmm2[6],xmm7[6],xmm2[7],xmm7[7]2206; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]2207; SSE-NEXT:    movdqa %xmm0, %xmm42208; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]2209; SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]2210; SSE-NEXT:    movdqa %xmm0, 496(%r8)2211; SSE-NEXT:    movdqa %xmm4, 480(%r8)2212; SSE-NEXT:    movdqa %xmm1, 464(%r8)2213; SSE-NEXT:    movdqa %xmm3, 448(%r8)2214; SSE-NEXT:    movdqa %xmm5, 432(%r8)2215; SSE-NEXT:    movdqa %xmm10, 416(%r8)2216; SSE-NEXT:    movdqa %xmm9, 400(%r8)2217; SSE-NEXT:    movdqa %xmm12, 384(%r8)2218; SSE-NEXT:    movdqa %xmm11, 368(%r8)2219; SSE-NEXT:    movdqa %xmm15, 352(%r8)2220; SSE-NEXT:    movdqa %xmm8, 336(%r8)2221; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2222; SSE-NEXT:    movaps %xmm0, 320(%r8)2223; SSE-NEXT:    movdqa %xmm13, 304(%r8)2224; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2225; SSE-NEXT:    movaps %xmm0, 288(%r8)2226; SSE-NEXT:    movdqa %xmm14, 272(%r8)2227; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2228; SSE-NEXT:    movaps %xmm0, 256(%r8)2229; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2230; SSE-NEXT:    movaps %xmm0, 240(%r8)2231; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2232; SSE-NEXT:    movaps %xmm0, 224(%r8)2233; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2234; SSE-NEXT:    movaps %xmm0, 208(%r8)2235; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2236; SSE-NEXT:    movaps %xmm0, 192(%r8)2237; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2238; SSE-NEXT:    movaps %xmm0, 176(%r8)2239; SSE-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload2240; SSE-NEXT:    movaps %xmm0, 160(%r8)2241; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2242; SSE-NEXT:    movaps %xmm0, 144(%r8)2243; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2244; SSE-NEXT:    movaps %xmm0, 128(%r8)2245; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2246; SSE-NEXT:    movaps %xmm0, 112(%r8)2247; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2248; SSE-NEXT:    movaps %xmm0, 96(%r8)2249; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2250; SSE-NEXT:    movaps %xmm0, 80(%r8)2251; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2252; SSE-NEXT:    movaps %xmm0, 64(%r8)2253; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2254; SSE-NEXT:    movaps %xmm0, 48(%r8)2255; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2256; SSE-NEXT:    movaps %xmm0, 32(%r8)2257; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2258; SSE-NEXT:    movaps %xmm0, 16(%r8)2259; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload2260; SSE-NEXT:    movaps %xmm0, (%r8)2261; SSE-NEXT:    addq $184, %rsp2262; SSE-NEXT:    retq2263;2264; AVX-LABEL: store_i16_stride4_vf64:2265; AVX:       # %bb.0:2266; AVX-NEXT:    subq $40, %rsp2267; AVX-NEXT:    vmovdqa 16(%rcx), %xmm12268; AVX-NEXT:    vmovdqa 32(%rcx), %xmm92269; AVX-NEXT:    vmovdqa 48(%rcx), %xmm62270; AVX-NEXT:    vmovdqa 16(%rdx), %xmm22271; AVX-NEXT:    vmovdqa 32(%rdx), %xmm102272; AVX-NEXT:    vmovdqa 48(%rdx), %xmm82273; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]2274; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[0,0,1,1]2275; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2276; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm02277; AVX-NEXT:    vmovdqa 16(%rsi), %xmm32278; AVX-NEXT:    vmovdqa 32(%rsi), %xmm112279; AVX-NEXT:    vmovdqa 16(%rdi), %xmm42280; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]2281; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero2282; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]2283; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm7, %ymm52284; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm5[0],ymm0[1],ymm5[2],ymm0[3],ymm5[4],ymm0[5],ymm5[6],ymm0[7]2285; AVX-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill2286; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]2287; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[0,0,1,1]2288; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]2289; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm12290; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]2291; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero2292; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]2293; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm22294; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7]2295; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2296; AVX-NEXT:    vmovdqa 64(%rcx), %xmm32297; AVX-NEXT:    vmovdqa 64(%rdx), %xmm42298; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]2299; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[0,0,1,1]2300; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]2301; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm22302; AVX-NEXT:    vmovdqa 64(%rsi), %xmm52303; AVX-NEXT:    vmovdqa 64(%rdi), %xmm72304; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]2305; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm13 = xmm12[0],zero,xmm12[1],zero2306; AVX-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[2,2,3,3]2307; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm13, %ymm122308; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm12[0],ymm2[1],ymm12[2],ymm2[3],ymm12[4],ymm2[5],ymm12[6],ymm2[7]2309; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2310; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]2311; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[0,0,1,1]2312; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,2,3,3]2313; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm32314; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]2315; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero2316; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]2317; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm42318; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7]2319; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2320; AVX-NEXT:    vmovdqa 80(%rcx), %xmm52321; AVX-NEXT:    vmovdqa 80(%rdx), %xmm72322; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]2323; AVX-NEXT:    vpshufd {{.*#+}} xmm12 = xmm4[0,0,1,1]2324; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]2325; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm12, %ymm42326; AVX-NEXT:    vmovdqa 80(%rsi), %xmm132327; AVX-NEXT:    vmovdqa 80(%rdi), %xmm142328; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]2329; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm15 = xmm12[0],zero,xmm12[1],zero2330; AVX-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[2,2,3,3]2331; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm15, %ymm152332; AVX-NEXT:    vmovdqa 48(%rsi), %xmm122333; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm15[0],ymm4[1],ymm15[2],ymm4[3],ymm15[4],ymm4[5],ymm15[6],ymm4[7]2334; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2335; AVX-NEXT:    vmovdqa 32(%rdi), %xmm152336; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]2337; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm5[0,0,1,1]2338; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]2339; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm7, %ymm52340; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]2341; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm13 = xmm7[0],zero,xmm7[1],zero2342; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]2343; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm13, %ymm72344; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm7[0],ymm5[1],ymm7[2],ymm5[3],ymm7[4],ymm5[5],ymm7[6],ymm5[7]2345; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]2346; AVX-NEXT:    vpshufd {{.*#+}} xmm13 = xmm7[0,0,1,1]2347; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]2348; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm13, %ymm72349; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm15[0],xmm11[0],xmm15[1],xmm11[1],xmm15[2],xmm11[2],xmm15[3],xmm11[3]2350; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm14 = xmm13[0],zero,xmm13[1],zero2351; AVX-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[2,2,3,3]2352; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm14, %ymm132353; AVX-NEXT:    vmovdqa 48(%rdi), %xmm142354; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm13[0],ymm7[1],ymm13[2],ymm7[3],ymm13[4],ymm7[5],ymm13[6],ymm7[7]2355; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]2356; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm9[0,0,1,1]2357; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]2358; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm10, %ymm92359; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm15[4],xmm11[4],xmm15[5],xmm11[5],xmm15[6],xmm11[6],xmm15[7],xmm11[7]2360; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm11 = xmm10[0],zero,xmm10[1],zero2361; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm10[2,2,3,3]2362; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm11, %ymm102363; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0],ymm9[1],ymm10[2],ymm9[3],ymm10[4],ymm9[5],ymm10[6],ymm9[7]2364; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]2365; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm10[0,0,1,1]2366; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm10[2,2,3,3]2367; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm11, %ymm102368; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm14[0],xmm12[0],xmm14[1],xmm12[1],xmm14[2],xmm12[2],xmm14[3],xmm12[3]2369; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm13 = xmm11[0],zero,xmm11[1],zero2370; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,2,3,3]2371; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm13, %ymm112372; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0],ymm10[1],ymm11[2],ymm10[3],ymm11[4],ymm10[5],ymm11[6],ymm10[7]2373; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]2374; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm6[0,0,1,1]2375; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]2376; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm8, %ymm62377; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm14[4],xmm12[4],xmm14[5],xmm12[5],xmm14[6],xmm12[6],xmm14[7],xmm12[7]2378; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm11 = xmm8[0],zero,xmm8[1],zero2379; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]2380; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm82381; AVX-NEXT:    vblendps {{.*#+}} ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7]2382; AVX-NEXT:    vmovdqa 96(%rcx), %xmm112383; AVX-NEXT:    vmovdqa 96(%rdx), %xmm122384; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]2385; AVX-NEXT:    vpshufd {{.*#+}} xmm13 = xmm8[0,0,1,1]2386; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]2387; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm13, %ymm82388; AVX-NEXT:    vmovdqa 96(%rsi), %xmm132389; AVX-NEXT:    vmovdqa 96(%rdi), %xmm142390; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]2391; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm15[0],zero,xmm15[1],zero2392; AVX-NEXT:    vpshufd {{.*#+}} xmm15 = xmm15[2,2,3,3]2393; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm0, %ymm02394; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm0[0],ymm8[1],ymm0[2],ymm8[3],ymm0[4],ymm8[5],ymm0[6],ymm8[7]2395; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]2396; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm0[0,0,1,1]2397; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2398; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm11, %ymm02399; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]2400; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm12 = xmm11[0],zero,xmm11[1],zero2401; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,2,3,3]2402; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm12, %ymm112403; AVX-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0],ymm0[1],ymm11[2],ymm0[3],ymm11[4],ymm0[5],ymm11[6],ymm0[7]2404; AVX-NEXT:    vmovdqa 112(%rcx), %xmm12405; AVX-NEXT:    vmovdqa 112(%rdx), %xmm132406; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]2407; AVX-NEXT:    vpshufd {{.*#+}} xmm14 = xmm12[0,0,1,1]2408; AVX-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[2,2,3,3]2409; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm14, %ymm122410; AVX-NEXT:    vmovdqa 112(%rsi), %xmm142411; AVX-NEXT:    vmovdqa 112(%rdi), %xmm152412; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]2413; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero2414; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2415; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm02416; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm0[0],ymm12[1],ymm0[2],ymm12[3],ymm0[4],ymm12[5],ymm0[6],ymm12[7]2417; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm13[4],xmm1[4],xmm13[5],xmm1[5],xmm13[6],xmm1[6],xmm13[7],xmm1[7]2418; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]2419; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2420; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm02421; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]2422; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero2423; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]2424; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm12425; AVX-NEXT:    vmovdqa (%rcx), %xmm22426; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]2427; AVX-NEXT:    vmovdqa (%rdx), %xmm12428; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]2429; AVX-NEXT:    vpshufd {{.*#+}} xmm14 = xmm13[0,0,1,1]2430; AVX-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[2,2,3,3]2431; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm14, %ymm132432; AVX-NEXT:    vmovdqa (%rsi), %xmm142433; AVX-NEXT:    vmovdqa (%rdi), %xmm152434; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]2435; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero2436; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2437; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm02438; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm13[1],ymm0[2],ymm13[3],ymm0[4],ymm13[5],ymm0[6],ymm13[7]2439; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]2440; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[0,0,1,1]2441; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]2442; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm12443; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]2444; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero2445; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]2446; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm22447; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7]2448; AVX-NEXT:    vmovaps %ymm1, 32(%r8)2449; AVX-NEXT:    vmovaps %ymm0, (%r8)2450; AVX-NEXT:    vmovaps %ymm4, 480(%r8)2451; AVX-NEXT:    vmovaps %ymm12, 448(%r8)2452; AVX-NEXT:    vmovaps %ymm11, 416(%r8)2453; AVX-NEXT:    vmovaps %ymm8, 384(%r8)2454; AVX-NEXT:    vmovaps %ymm6, 224(%r8)2455; AVX-NEXT:    vmovaps %ymm10, 192(%r8)2456; AVX-NEXT:    vmovaps %ymm9, 160(%r8)2457; AVX-NEXT:    vmovaps %ymm7, 128(%r8)2458; AVX-NEXT:    vmovaps %ymm5, 352(%r8)2459; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2460; AVX-NEXT:    vmovaps %ymm0, 320(%r8)2461; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2462; AVX-NEXT:    vmovaps %ymm0, 288(%r8)2463; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2464; AVX-NEXT:    vmovaps %ymm0, 256(%r8)2465; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2466; AVX-NEXT:    vmovaps %ymm0, 64(%r8)2467; AVX-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload2468; AVX-NEXT:    vmovaps %ymm0, 96(%r8)2469; AVX-NEXT:    addq $40, %rsp2470; AVX-NEXT:    vzeroupper2471; AVX-NEXT:    retq2472;2473; AVX2-LABEL: store_i16_stride4_vf64:2474; AVX2:       # %bb.0:2475; AVX2-NEXT:    subq $40, %rsp2476; AVX2-NEXT:    vmovdqa 16(%rcx), %xmm12477; AVX2-NEXT:    vmovdqa 32(%rcx), %xmm92478; AVX2-NEXT:    vmovdqa 48(%rcx), %xmm62479; AVX2-NEXT:    vmovdqa 16(%rdx), %xmm22480; AVX2-NEXT:    vmovdqa 32(%rdx), %xmm102481; AVX2-NEXT:    vmovdqa 48(%rdx), %xmm82482; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]2483; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[0,0,1,1]2484; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2485; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm3, %ymm02486; AVX2-NEXT:    vmovdqa 16(%rsi), %xmm32487; AVX2-NEXT:    vmovdqa 32(%rsi), %xmm112488; AVX2-NEXT:    vmovdqa 16(%rdi), %xmm42489; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]2490; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero2491; AVX2-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]2492; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm7, %ymm52493; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0],ymm0[1],ymm5[2],ymm0[3],ymm5[4],ymm0[5],ymm5[6],ymm0[7]2494; AVX2-NEXT:    vmovdqu %ymm0, (%rsp) # 32-byte Spill2495; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]2496; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[0,0,1,1]2497; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]2498; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm12499; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]2500; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero2501; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]2502; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm22503; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7]2504; AVX2-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2505; AVX2-NEXT:    vmovdqa 64(%rcx), %xmm32506; AVX2-NEXT:    vmovdqa 64(%rdx), %xmm42507; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]2508; AVX2-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[0,0,1,1]2509; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]2510; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm5, %ymm22511; AVX2-NEXT:    vmovdqa 64(%rsi), %xmm52512; AVX2-NEXT:    vmovdqa 64(%rdi), %xmm72513; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]2514; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm13 = xmm12[0],zero,xmm12[1],zero2515; AVX2-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[2,2,3,3]2516; AVX2-NEXT:    vinserti128 $1, %xmm12, %ymm13, %ymm122517; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm12[0],ymm2[1],ymm12[2],ymm2[3],ymm12[4],ymm2[5],ymm12[6],ymm2[7]2518; AVX2-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2519; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]2520; AVX2-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[0,0,1,1]2521; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,2,3,3]2522; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm4, %ymm32523; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]2524; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero2525; AVX2-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]2526; AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm42527; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7]2528; AVX2-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2529; AVX2-NEXT:    vmovdqa 80(%rcx), %xmm52530; AVX2-NEXT:    vmovdqa 80(%rdx), %xmm72531; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]2532; AVX2-NEXT:    vpshufd {{.*#+}} xmm12 = xmm4[0,0,1,1]2533; AVX2-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]2534; AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm12, %ymm42535; AVX2-NEXT:    vmovdqa 80(%rsi), %xmm132536; AVX2-NEXT:    vmovdqa 80(%rdi), %xmm142537; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]2538; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm15 = xmm12[0],zero,xmm12[1],zero2539; AVX2-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[2,2,3,3]2540; AVX2-NEXT:    vinserti128 $1, %xmm12, %ymm15, %ymm152541; AVX2-NEXT:    vmovdqa 48(%rsi), %xmm122542; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm15[0],ymm4[1],ymm15[2],ymm4[3],ymm15[4],ymm4[5],ymm15[6],ymm4[7]2543; AVX2-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2544; AVX2-NEXT:    vmovdqa 32(%rdi), %xmm152545; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]2546; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm5[0,0,1,1]2547; AVX2-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]2548; AVX2-NEXT:    vinserti128 $1, %xmm5, %ymm7, %ymm52549; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]2550; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm13 = xmm7[0],zero,xmm7[1],zero2551; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]2552; AVX2-NEXT:    vinserti128 $1, %xmm7, %ymm13, %ymm72553; AVX2-NEXT:    vpblendd {{.*#+}} ymm5 = ymm7[0],ymm5[1],ymm7[2],ymm5[3],ymm7[4],ymm5[5],ymm7[6],ymm5[7]2554; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]2555; AVX2-NEXT:    vpshufd {{.*#+}} xmm13 = xmm7[0,0,1,1]2556; AVX2-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]2557; AVX2-NEXT:    vinserti128 $1, %xmm7, %ymm13, %ymm72558; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm15[0],xmm11[0],xmm15[1],xmm11[1],xmm15[2],xmm11[2],xmm15[3],xmm11[3]2559; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm14 = xmm13[0],zero,xmm13[1],zero2560; AVX2-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[2,2,3,3]2561; AVX2-NEXT:    vinserti128 $1, %xmm13, %ymm14, %ymm132562; AVX2-NEXT:    vmovdqa 48(%rdi), %xmm142563; AVX2-NEXT:    vpblendd {{.*#+}} ymm7 = ymm13[0],ymm7[1],ymm13[2],ymm7[3],ymm13[4],ymm7[5],ymm13[6],ymm7[7]2564; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]2565; AVX2-NEXT:    vpshufd {{.*#+}} xmm10 = xmm9[0,0,1,1]2566; AVX2-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]2567; AVX2-NEXT:    vinserti128 $1, %xmm9, %ymm10, %ymm92568; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm15[4],xmm11[4],xmm15[5],xmm11[5],xmm15[6],xmm11[6],xmm15[7],xmm11[7]2569; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm11 = xmm10[0],zero,xmm10[1],zero2570; AVX2-NEXT:    vpshufd {{.*#+}} xmm10 = xmm10[2,2,3,3]2571; AVX2-NEXT:    vinserti128 $1, %xmm10, %ymm11, %ymm102572; AVX2-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0],ymm9[1],ymm10[2],ymm9[3],ymm10[4],ymm9[5],ymm10[6],ymm9[7]2573; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]2574; AVX2-NEXT:    vpshufd {{.*#+}} xmm11 = xmm10[0,0,1,1]2575; AVX2-NEXT:    vpshufd {{.*#+}} xmm10 = xmm10[2,2,3,3]2576; AVX2-NEXT:    vinserti128 $1, %xmm10, %ymm11, %ymm102577; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm14[0],xmm12[0],xmm14[1],xmm12[1],xmm14[2],xmm12[2],xmm14[3],xmm12[3]2578; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm13 = xmm11[0],zero,xmm11[1],zero2579; AVX2-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,2,3,3]2580; AVX2-NEXT:    vinserti128 $1, %xmm11, %ymm13, %ymm112581; AVX2-NEXT:    vpblendd {{.*#+}} ymm10 = ymm11[0],ymm10[1],ymm11[2],ymm10[3],ymm11[4],ymm10[5],ymm11[6],ymm10[7]2582; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]2583; AVX2-NEXT:    vpshufd {{.*#+}} xmm8 = xmm6[0,0,1,1]2584; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]2585; AVX2-NEXT:    vinserti128 $1, %xmm6, %ymm8, %ymm62586; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm14[4],xmm12[4],xmm14[5],xmm12[5],xmm14[6],xmm12[6],xmm14[7],xmm12[7]2587; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm11 = xmm8[0],zero,xmm8[1],zero2588; AVX2-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]2589; AVX2-NEXT:    vinserti128 $1, %xmm8, %ymm11, %ymm82590; AVX2-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7]2591; AVX2-NEXT:    vmovdqa 96(%rcx), %xmm112592; AVX2-NEXT:    vmovdqa 96(%rdx), %xmm122593; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]2594; AVX2-NEXT:    vpshufd {{.*#+}} xmm13 = xmm8[0,0,1,1]2595; AVX2-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]2596; AVX2-NEXT:    vinserti128 $1, %xmm8, %ymm13, %ymm82597; AVX2-NEXT:    vmovdqa 96(%rsi), %xmm132598; AVX2-NEXT:    vmovdqa 96(%rdi), %xmm142599; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]2600; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm15[0],zero,xmm15[1],zero2601; AVX2-NEXT:    vpshufd {{.*#+}} xmm15 = xmm15[2,2,3,3]2602; AVX2-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm02603; AVX2-NEXT:    vpblendd {{.*#+}} ymm8 = ymm0[0],ymm8[1],ymm0[2],ymm8[3],ymm0[4],ymm8[5],ymm0[6],ymm8[7]2604; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]2605; AVX2-NEXT:    vpshufd {{.*#+}} xmm11 = xmm0[0,0,1,1]2606; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2607; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm11, %ymm02608; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]2609; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm12 = xmm11[0],zero,xmm11[1],zero2610; AVX2-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,2,3,3]2611; AVX2-NEXT:    vinserti128 $1, %xmm11, %ymm12, %ymm112612; AVX2-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0],ymm0[1],ymm11[2],ymm0[3],ymm11[4],ymm0[5],ymm11[6],ymm0[7]2613; AVX2-NEXT:    vmovdqa 112(%rcx), %xmm12614; AVX2-NEXT:    vmovdqa 112(%rdx), %xmm132615; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]2616; AVX2-NEXT:    vpshufd {{.*#+}} xmm14 = xmm12[0,0,1,1]2617; AVX2-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[2,2,3,3]2618; AVX2-NEXT:    vinserti128 $1, %xmm12, %ymm14, %ymm122619; AVX2-NEXT:    vmovdqa 112(%rsi), %xmm142620; AVX2-NEXT:    vmovdqa 112(%rdi), %xmm152621; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]2622; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero2623; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2624; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm02625; AVX2-NEXT:    vpblendd {{.*#+}} ymm12 = ymm0[0],ymm12[1],ymm0[2],ymm12[3],ymm0[4],ymm12[5],ymm0[6],ymm12[7]2626; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm13[4],xmm1[4],xmm13[5],xmm1[5],xmm13[6],xmm1[6],xmm13[7],xmm1[7]2627; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]2628; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2629; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm02630; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]2631; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero2632; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]2633; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm12634; AVX2-NEXT:    vmovdqa (%rcx), %xmm22635; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]2636; AVX2-NEXT:    vmovdqa (%rdx), %xmm12637; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]2638; AVX2-NEXT:    vpshufd {{.*#+}} xmm14 = xmm13[0,0,1,1]2639; AVX2-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[2,2,3,3]2640; AVX2-NEXT:    vinserti128 $1, %xmm13, %ymm14, %ymm132641; AVX2-NEXT:    vmovdqa (%rsi), %xmm142642; AVX2-NEXT:    vmovdqa (%rdi), %xmm152643; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]2644; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero2645; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2646; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm3, %ymm02647; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm13[1],ymm0[2],ymm13[3],ymm0[4],ymm13[5],ymm0[6],ymm13[7]2648; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]2649; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[0,0,1,1]2650; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]2651; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm12652; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]2653; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero2654; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]2655; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm22656; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7]2657; AVX2-NEXT:    vmovdqa %ymm1, 32(%r8)2658; AVX2-NEXT:    vmovdqa %ymm0, (%r8)2659; AVX2-NEXT:    vmovdqa %ymm4, 480(%r8)2660; AVX2-NEXT:    vmovdqa %ymm12, 448(%r8)2661; AVX2-NEXT:    vmovdqa %ymm11, 416(%r8)2662; AVX2-NEXT:    vmovdqa %ymm8, 384(%r8)2663; AVX2-NEXT:    vmovdqa %ymm6, 224(%r8)2664; AVX2-NEXT:    vmovdqa %ymm10, 192(%r8)2665; AVX2-NEXT:    vmovdqa %ymm9, 160(%r8)2666; AVX2-NEXT:    vmovdqa %ymm7, 128(%r8)2667; AVX2-NEXT:    vmovdqa %ymm5, 352(%r8)2668; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2669; AVX2-NEXT:    vmovaps %ymm0, 320(%r8)2670; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2671; AVX2-NEXT:    vmovaps %ymm0, 288(%r8)2672; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2673; AVX2-NEXT:    vmovaps %ymm0, 256(%r8)2674; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2675; AVX2-NEXT:    vmovaps %ymm0, 64(%r8)2676; AVX2-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload2677; AVX2-NEXT:    vmovaps %ymm0, 96(%r8)2678; AVX2-NEXT:    addq $40, %rsp2679; AVX2-NEXT:    vzeroupper2680; AVX2-NEXT:    retq2681;2682; AVX2-FP-LABEL: store_i16_stride4_vf64:2683; AVX2-FP:       # %bb.0:2684; AVX2-FP-NEXT:    subq $40, %rsp2685; AVX2-FP-NEXT:    vmovdqa 16(%rcx), %xmm12686; AVX2-FP-NEXT:    vmovdqa 32(%rcx), %xmm92687; AVX2-FP-NEXT:    vmovdqa 48(%rcx), %xmm62688; AVX2-FP-NEXT:    vmovdqa 16(%rdx), %xmm22689; AVX2-FP-NEXT:    vmovdqa 32(%rdx), %xmm102690; AVX2-FP-NEXT:    vmovdqa 48(%rdx), %xmm82691; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]2692; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[0,0,1,1]2693; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2694; AVX2-FP-NEXT:    vinserti128 $1, %xmm0, %ymm3, %ymm02695; AVX2-FP-NEXT:    vmovdqa 16(%rsi), %xmm32696; AVX2-FP-NEXT:    vmovdqa 32(%rsi), %xmm112697; AVX2-FP-NEXT:    vmovdqa 16(%rdi), %xmm42698; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]2699; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero2700; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]2701; AVX2-FP-NEXT:    vinserti128 $1, %xmm5, %ymm7, %ymm52702; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0],ymm0[1],ymm5[2],ymm0[3],ymm5[4],ymm0[5],ymm5[6],ymm0[7]2703; AVX2-FP-NEXT:    vmovdqu %ymm0, (%rsp) # 32-byte Spill2704; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]2705; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[0,0,1,1]2706; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]2707; AVX2-FP-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm12708; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]2709; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero2710; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]2711; AVX2-FP-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm22712; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7]2713; AVX2-FP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2714; AVX2-FP-NEXT:    vmovdqa 64(%rcx), %xmm32715; AVX2-FP-NEXT:    vmovdqa 64(%rdx), %xmm42716; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]2717; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[0,0,1,1]2718; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]2719; AVX2-FP-NEXT:    vinserti128 $1, %xmm2, %ymm5, %ymm22720; AVX2-FP-NEXT:    vmovdqa 64(%rsi), %xmm52721; AVX2-FP-NEXT:    vmovdqa 64(%rdi), %xmm72722; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]2723; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm13 = xmm12[0],zero,xmm12[1],zero2724; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[2,2,3,3]2725; AVX2-FP-NEXT:    vinserti128 $1, %xmm12, %ymm13, %ymm122726; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm12[0],ymm2[1],ymm12[2],ymm2[3],ymm12[4],ymm2[5],ymm12[6],ymm2[7]2727; AVX2-FP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2728; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]2729; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[0,0,1,1]2730; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,2,3,3]2731; AVX2-FP-NEXT:    vinserti128 $1, %xmm3, %ymm4, %ymm32732; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]2733; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero2734; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]2735; AVX2-FP-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm42736; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0],ymm3[1],ymm4[2],ymm3[3],ymm4[4],ymm3[5],ymm4[6],ymm3[7]2737; AVX2-FP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2738; AVX2-FP-NEXT:    vmovdqa 80(%rcx), %xmm52739; AVX2-FP-NEXT:    vmovdqa 80(%rdx), %xmm72740; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]2741; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm12 = xmm4[0,0,1,1]2742; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]2743; AVX2-FP-NEXT:    vinserti128 $1, %xmm4, %ymm12, %ymm42744; AVX2-FP-NEXT:    vmovdqa 80(%rsi), %xmm132745; AVX2-FP-NEXT:    vmovdqa 80(%rdi), %xmm142746; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]2747; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm15 = xmm12[0],zero,xmm12[1],zero2748; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[2,2,3,3]2749; AVX2-FP-NEXT:    vinserti128 $1, %xmm12, %ymm15, %ymm152750; AVX2-FP-NEXT:    vmovdqa 48(%rsi), %xmm122751; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm15[0],ymm4[1],ymm15[2],ymm4[3],ymm15[4],ymm4[5],ymm15[6],ymm4[7]2752; AVX2-FP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2753; AVX2-FP-NEXT:    vmovdqa 32(%rdi), %xmm152754; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]2755; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm7 = xmm5[0,0,1,1]2756; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]2757; AVX2-FP-NEXT:    vinserti128 $1, %xmm5, %ymm7, %ymm52758; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]2759; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm13 = xmm7[0],zero,xmm7[1],zero2760; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]2761; AVX2-FP-NEXT:    vinserti128 $1, %xmm7, %ymm13, %ymm72762; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm7[0],ymm5[1],ymm7[2],ymm5[3],ymm7[4],ymm5[5],ymm7[6],ymm5[7]2763; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]2764; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm13 = xmm7[0,0,1,1]2765; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3]2766; AVX2-FP-NEXT:    vinserti128 $1, %xmm7, %ymm13, %ymm72767; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm15[0],xmm11[0],xmm15[1],xmm11[1],xmm15[2],xmm11[2],xmm15[3],xmm11[3]2768; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm14 = xmm13[0],zero,xmm13[1],zero2769; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[2,2,3,3]2770; AVX2-FP-NEXT:    vinserti128 $1, %xmm13, %ymm14, %ymm132771; AVX2-FP-NEXT:    vmovdqa 48(%rdi), %xmm142772; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm13[0],ymm7[1],ymm13[2],ymm7[3],ymm13[4],ymm7[5],ymm13[6],ymm7[7]2773; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]2774; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm10 = xmm9[0,0,1,1]2775; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]2776; AVX2-FP-NEXT:    vinserti128 $1, %xmm9, %ymm10, %ymm92777; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm15[4],xmm11[4],xmm15[5],xmm11[5],xmm15[6],xmm11[6],xmm15[7],xmm11[7]2778; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm11 = xmm10[0],zero,xmm10[1],zero2779; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm10 = xmm10[2,2,3,3]2780; AVX2-FP-NEXT:    vinserti128 $1, %xmm10, %ymm11, %ymm102781; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0],ymm9[1],ymm10[2],ymm9[3],ymm10[4],ymm9[5],ymm10[6],ymm9[7]2782; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]2783; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm11 = xmm10[0,0,1,1]2784; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm10 = xmm10[2,2,3,3]2785; AVX2-FP-NEXT:    vinserti128 $1, %xmm10, %ymm11, %ymm102786; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm14[0],xmm12[0],xmm14[1],xmm12[1],xmm14[2],xmm12[2],xmm14[3],xmm12[3]2787; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm13 = xmm11[0],zero,xmm11[1],zero2788; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,2,3,3]2789; AVX2-FP-NEXT:    vinserti128 $1, %xmm11, %ymm13, %ymm112790; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm11[0],ymm10[1],ymm11[2],ymm10[3],ymm11[4],ymm10[5],ymm11[6],ymm10[7]2791; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]2792; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm6[0,0,1,1]2793; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]2794; AVX2-FP-NEXT:    vinserti128 $1, %xmm6, %ymm8, %ymm62795; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm14[4],xmm12[4],xmm14[5],xmm12[5],xmm14[6],xmm12[6],xmm14[7],xmm12[7]2796; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm11 = xmm8[0],zero,xmm8[1],zero2797; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]2798; AVX2-FP-NEXT:    vinserti128 $1, %xmm8, %ymm11, %ymm82799; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0],ymm6[1],ymm8[2],ymm6[3],ymm8[4],ymm6[5],ymm8[6],ymm6[7]2800; AVX2-FP-NEXT:    vmovdqa 96(%rcx), %xmm112801; AVX2-FP-NEXT:    vmovdqa 96(%rdx), %xmm122802; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]2803; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm13 = xmm8[0,0,1,1]2804; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]2805; AVX2-FP-NEXT:    vinserti128 $1, %xmm8, %ymm13, %ymm82806; AVX2-FP-NEXT:    vmovdqa 96(%rsi), %xmm132807; AVX2-FP-NEXT:    vmovdqa 96(%rdi), %xmm142808; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]2809; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm15[0],zero,xmm15[1],zero2810; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm15 = xmm15[2,2,3,3]2811; AVX2-FP-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm02812; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm8 = ymm0[0],ymm8[1],ymm0[2],ymm8[3],ymm0[4],ymm8[5],ymm0[6],ymm8[7]2813; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]2814; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm11 = xmm0[0,0,1,1]2815; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2816; AVX2-FP-NEXT:    vinserti128 $1, %xmm0, %ymm11, %ymm02817; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]2818; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm12 = xmm11[0],zero,xmm11[1],zero2819; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,2,3,3]2820; AVX2-FP-NEXT:    vinserti128 $1, %xmm11, %ymm12, %ymm112821; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0],ymm0[1],ymm11[2],ymm0[3],ymm11[4],ymm0[5],ymm11[6],ymm0[7]2822; AVX2-FP-NEXT:    vmovdqa 112(%rcx), %xmm12823; AVX2-FP-NEXT:    vmovdqa 112(%rdx), %xmm132824; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]2825; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm14 = xmm12[0,0,1,1]2826; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[2,2,3,3]2827; AVX2-FP-NEXT:    vinserti128 $1, %xmm12, %ymm14, %ymm122828; AVX2-FP-NEXT:    vmovdqa 112(%rsi), %xmm142829; AVX2-FP-NEXT:    vmovdqa 112(%rdi), %xmm152830; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]2831; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero2832; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2833; AVX2-FP-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm02834; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm12 = ymm0[0],ymm12[1],ymm0[2],ymm12[3],ymm0[4],ymm12[5],ymm0[6],ymm12[7]2835; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm13[4],xmm1[4],xmm13[5],xmm1[5],xmm13[6],xmm1[6],xmm13[7],xmm1[7]2836; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1]2837; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2838; AVX2-FP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm02839; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]2840; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero2841; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]2842; AVX2-FP-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm12843; AVX2-FP-NEXT:    vmovdqa (%rcx), %xmm22844; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]2845; AVX2-FP-NEXT:    vmovdqa (%rdx), %xmm12846; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]2847; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm14 = xmm13[0,0,1,1]2848; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[2,2,3,3]2849; AVX2-FP-NEXT:    vinserti128 $1, %xmm13, %ymm14, %ymm132850; AVX2-FP-NEXT:    vmovdqa (%rsi), %xmm142851; AVX2-FP-NEXT:    vmovdqa (%rdi), %xmm152852; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]2853; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero2854; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2855; AVX2-FP-NEXT:    vinserti128 $1, %xmm0, %ymm3, %ymm02856; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm13[1],ymm0[2],ymm13[3],ymm0[4],ymm13[5],ymm0[6],ymm13[7]2857; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]2858; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[0,0,1,1]2859; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]2860; AVX2-FP-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm12861; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]2862; AVX2-FP-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero2863; AVX2-FP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]2864; AVX2-FP-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm22865; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7]2866; AVX2-FP-NEXT:    vmovdqa %ymm1, 32(%r8)2867; AVX2-FP-NEXT:    vmovdqa %ymm0, (%r8)2868; AVX2-FP-NEXT:    vmovdqa %ymm4, 480(%r8)2869; AVX2-FP-NEXT:    vmovdqa %ymm12, 448(%r8)2870; AVX2-FP-NEXT:    vmovdqa %ymm11, 416(%r8)2871; AVX2-FP-NEXT:    vmovdqa %ymm8, 384(%r8)2872; AVX2-FP-NEXT:    vmovdqa %ymm6, 224(%r8)2873; AVX2-FP-NEXT:    vmovdqa %ymm10, 192(%r8)2874; AVX2-FP-NEXT:    vmovdqa %ymm9, 160(%r8)2875; AVX2-FP-NEXT:    vmovdqa %ymm7, 128(%r8)2876; AVX2-FP-NEXT:    vmovdqa %ymm5, 352(%r8)2877; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2878; AVX2-FP-NEXT:    vmovaps %ymm0, 320(%r8)2879; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2880; AVX2-FP-NEXT:    vmovaps %ymm0, 288(%r8)2881; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2882; AVX2-FP-NEXT:    vmovaps %ymm0, 256(%r8)2883; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2884; AVX2-FP-NEXT:    vmovaps %ymm0, 64(%r8)2885; AVX2-FP-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload2886; AVX2-FP-NEXT:    vmovaps %ymm0, 96(%r8)2887; AVX2-FP-NEXT:    addq $40, %rsp2888; AVX2-FP-NEXT:    vzeroupper2889; AVX2-FP-NEXT:    retq2890;2891; AVX2-FCP-LABEL: store_i16_stride4_vf64:2892; AVX2-FCP:       # %bb.0:2893; AVX2-FCP-NEXT:    subq $40, %rsp2894; AVX2-FCP-NEXT:    vmovdqa 16(%rsi), %xmm12895; AVX2-FCP-NEXT:    vmovdqa 32(%rsi), %xmm102896; AVX2-FCP-NEXT:    vmovdqa 48(%rsi), %xmm72897; AVX2-FCP-NEXT:    vmovdqa 16(%rdi), %xmm32898; AVX2-FCP-NEXT:    vmovdqa 32(%rdi), %xmm112899; AVX2-FCP-NEXT:    vmovdqa 48(%rdi), %xmm92900; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]2901; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero2902; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2903; AVX2-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm02904; AVX2-FCP-NEXT:    vmovdqa 16(%rcx), %xmm42905; AVX2-FCP-NEXT:    vmovdqa 32(%rcx), %xmm122906; AVX2-FCP-NEXT:    vmovdqa 16(%rdx), %xmm52907; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]2908; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm2 = [0,0,1,1,2,2,3,3]2909; AVX2-FCP-NEXT:    vpermd %ymm6, %ymm2, %ymm62910; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm6[1],ymm0[2],ymm6[3],ymm0[4],ymm6[5],ymm0[6],ymm6[7]2911; AVX2-FCP-NEXT:    vmovdqu %ymm0, (%rsp) # 32-byte Spill2912; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]2913; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero2914; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]2915; AVX2-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm3, %ymm12916; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]2917; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm2, %ymm32918; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm3[1],ymm1[2],ymm3[3],ymm1[4],ymm3[5],ymm1[6],ymm3[7]2919; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2920; AVX2-FCP-NEXT:    vmovdqa 64(%rsi), %xmm42921; AVX2-FCP-NEXT:    vmovdqa 64(%rdi), %xmm52922; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]2923; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm3[0],zero,xmm3[1],zero2924; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,2,3,3]2925; AVX2-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm6, %ymm32926; AVX2-FCP-NEXT:    vmovdqa 64(%rcx), %xmm62927; AVX2-FCP-NEXT:    vmovdqa 64(%rdx), %xmm82928; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]2929; AVX2-FCP-NEXT:    vpermd %ymm13, %ymm2, %ymm132930; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0],ymm13[1],ymm3[2],ymm13[3],ymm3[4],ymm13[5],ymm3[6],ymm13[7]2931; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2932; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]2933; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero2934; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]2935; AVX2-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm42936; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]2937; AVX2-FCP-NEXT:    vpermd %ymm5, %ymm2, %ymm52938; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4],ymm5[5],ymm4[6],ymm5[7]2939; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2940; AVX2-FCP-NEXT:    vmovdqa 80(%rsi), %xmm62941; AVX2-FCP-NEXT:    vmovdqa 80(%rdi), %xmm82942; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]2943; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm13 = xmm5[0],zero,xmm5[1],zero2944; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,2,3,3]2945; AVX2-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm13, %ymm52946; AVX2-FCP-NEXT:    vmovdqa 80(%rcx), %xmm142947; AVX2-FCP-NEXT:    vmovdqa 80(%rdx), %xmm152948; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]2949; AVX2-FCP-NEXT:    vpermd %ymm13, %ymm2, %ymm132950; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0],ymm13[1],ymm5[2],ymm13[3],ymm5[4],ymm13[5],ymm5[6],ymm13[7]2951; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2952; AVX2-FCP-NEXT:    vmovdqa 48(%rcx), %xmm132953; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]2954; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm6[0],zero,xmm6[1],zero2955; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]2956; AVX2-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm8, %ymm62957; AVX2-FCP-NEXT:    vmovdqa 32(%rdx), %xmm02958; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]2959; AVX2-FCP-NEXT:    vpermd %ymm8, %ymm2, %ymm82960; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0],ymm8[1],ymm6[2],ymm8[3],ymm6[4],ymm8[5],ymm6[6],ymm8[7]2961; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]2962; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm14 = xmm8[0],zero,xmm8[1],zero2963; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]2964; AVX2-FCP-NEXT:    vinserti128 $1, %xmm8, %ymm14, %ymm82965; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm0[0],xmm12[0],xmm0[1],xmm12[1],xmm0[2],xmm12[2],xmm0[3],xmm12[3]2966; AVX2-FCP-NEXT:    vpermd %ymm14, %ymm2, %ymm142967; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0],ymm14[1],ymm8[2],ymm14[3],ymm8[4],ymm14[5],ymm8[6],ymm14[7]2968; AVX2-FCP-NEXT:    vmovdqa 48(%rdx), %xmm142969; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]2970; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm11 = xmm10[0],zero,xmm10[1],zero2971; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm10 = xmm10[2,2,3,3]2972; AVX2-FCP-NEXT:    vinserti128 $1, %xmm10, %ymm11, %ymm102973; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm12[4],xmm0[5],xmm12[5],xmm0[6],xmm12[6],xmm0[7],xmm12[7]2974; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm2, %ymm02975; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0],ymm0[1],ymm10[2],ymm0[3],ymm10[4],ymm0[5],ymm10[6],ymm0[7]2976; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]2977; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm11 = xmm0[0],zero,xmm0[1],zero2978; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2979; AVX2-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm11, %ymm02980; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]2981; AVX2-FCP-NEXT:    vpermd %ymm11, %ymm2, %ymm112982; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm11 = ymm0[0],ymm11[1],ymm0[2],ymm11[3],ymm0[4],ymm11[5],ymm0[6],ymm11[7]2983; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm9[4],xmm7[4],xmm9[5],xmm7[5],xmm9[6],xmm7[6],xmm9[7],xmm7[7]2984; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero2985; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]2986; AVX2-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm7, %ymm02987; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]2988; AVX2-FCP-NEXT:    vpermd %ymm7, %ymm2, %ymm72989; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm0[0],ymm7[1],ymm0[2],ymm7[3],ymm0[4],ymm7[5],ymm0[6],ymm7[7]2990; AVX2-FCP-NEXT:    vmovdqa 96(%rsi), %xmm02991; AVX2-FCP-NEXT:    vmovdqa 96(%rdi), %xmm122992; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm12[0],xmm0[0],xmm12[1],xmm0[1],xmm12[2],xmm0[2],xmm12[3],xmm0[3]2993; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm13 = xmm9[0],zero,xmm9[1],zero2994; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]2995; AVX2-FCP-NEXT:    vinserti128 $1, %xmm9, %ymm13, %ymm92996; AVX2-FCP-NEXT:    vmovdqa 96(%rcx), %xmm132997; AVX2-FCP-NEXT:    vmovdqa 96(%rdx), %xmm142998; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]2999; AVX2-FCP-NEXT:    vpermd %ymm15, %ymm2, %ymm153000; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0],ymm15[1],ymm9[2],ymm15[3],ymm9[4],ymm15[5],ymm9[6],ymm15[7]3001; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm12[4],xmm0[4],xmm12[5],xmm0[5],xmm12[6],xmm0[6],xmm12[7],xmm0[7]3002; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm12 = xmm0[0],zero,xmm0[1],zero3003; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3004; AVX2-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm12, %ymm03005; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm12 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]3006; AVX2-FCP-NEXT:    vpermd %ymm12, %ymm2, %ymm123007; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm12 = ymm0[0],ymm12[1],ymm0[2],ymm12[3],ymm0[4],ymm12[5],ymm0[6],ymm12[7]3008; AVX2-FCP-NEXT:    vmovdqa 112(%rsi), %xmm13009; AVX2-FCP-NEXT:    vmovdqa 112(%rdi), %xmm143010; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm1[0],xmm14[1],xmm1[1],xmm14[2],xmm1[2],xmm14[3],xmm1[3]3011; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm15 = xmm13[0],zero,xmm13[1],zero3012; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[2,2,3,3]3013; AVX2-FCP-NEXT:    vinserti128 $1, %xmm13, %ymm15, %ymm133014; AVX2-FCP-NEXT:    vmovdqa 112(%rcx), %xmm153015; AVX2-FCP-NEXT:    vmovdqa 112(%rdx), %xmm03016; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]3017; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm2, %ymm33018; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm13 = ymm13[0],ymm3[1],ymm13[2],ymm3[3],ymm13[4],ymm3[5],ymm13[6],ymm3[7]3019; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm14[4],xmm1[4],xmm14[5],xmm1[5],xmm14[6],xmm1[6],xmm14[7],xmm1[7]3020; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero3021; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]3022; AVX2-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm3, %ymm13023; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm15[4],xmm0[5],xmm15[5],xmm0[6],xmm15[6],xmm0[7],xmm15[7]3024; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm2, %ymm03025; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]3026; AVX2-FCP-NEXT:    vmovdqa (%rsi), %xmm13027; AVX2-FCP-NEXT:    vmovdqa (%rdi), %xmm33028; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]3029; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm15 = xmm14[0],zero,xmm14[1],zero3030; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm14 = xmm14[2,2,3,3]3031; AVX2-FCP-NEXT:    vinserti128 $1, %xmm14, %ymm15, %ymm143032; AVX2-FCP-NEXT:    vmovdqa (%rcx), %xmm153033; AVX2-FCP-NEXT:    vmovdqa (%rdx), %xmm03034; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]3035; AVX2-FCP-NEXT:    vpermd %ymm5, %ymm2, %ymm53036; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm14[0],ymm5[1],ymm14[2],ymm5[3],ymm14[4],ymm5[5],ymm14[6],ymm5[7]3037; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]3038; AVX2-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero3039; AVX2-FCP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]3040; AVX2-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm3, %ymm13041; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm15[4],xmm0[5],xmm15[5],xmm0[6],xmm15[6],xmm0[7],xmm15[7]3042; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm2, %ymm03043; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]3044; AVX2-FCP-NEXT:    vmovdqa %ymm0, 32(%r8)3045; AVX2-FCP-NEXT:    vmovdqa %ymm5, (%r8)3046; AVX2-FCP-NEXT:    vmovdqa %ymm4, 480(%r8)3047; AVX2-FCP-NEXT:    vmovdqa %ymm13, 448(%r8)3048; AVX2-FCP-NEXT:    vmovdqa %ymm12, 416(%r8)3049; AVX2-FCP-NEXT:    vmovdqa %ymm9, 384(%r8)3050; AVX2-FCP-NEXT:    vmovdqa %ymm7, 224(%r8)3051; AVX2-FCP-NEXT:    vmovdqa %ymm11, 192(%r8)3052; AVX2-FCP-NEXT:    vmovdqa %ymm10, 160(%r8)3053; AVX2-FCP-NEXT:    vmovdqa %ymm8, 128(%r8)3054; AVX2-FCP-NEXT:    vmovdqa %ymm6, 352(%r8)3055; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3056; AVX2-FCP-NEXT:    vmovaps %ymm0, 320(%r8)3057; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3058; AVX2-FCP-NEXT:    vmovaps %ymm0, 288(%r8)3059; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3060; AVX2-FCP-NEXT:    vmovaps %ymm0, 256(%r8)3061; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3062; AVX2-FCP-NEXT:    vmovaps %ymm0, 64(%r8)3063; AVX2-FCP-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload3064; AVX2-FCP-NEXT:    vmovaps %ymm0, 96(%r8)3065; AVX2-FCP-NEXT:    addq $40, %rsp3066; AVX2-FCP-NEXT:    vzeroupper3067; AVX2-FCP-NEXT:    retq3068;3069; AVX512-LABEL: store_i16_stride4_vf64:3070; AVX512:       # %bb.0:3071; AVX512-NEXT:    vmovdqa64 (%rsi), %xmm193072; AVX512-NEXT:    vmovdqa 16(%rsi), %xmm03073; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm113074; AVX512-NEXT:    vmovdqa 48(%rsi), %xmm63075; AVX512-NEXT:    vmovdqa64 (%rdi), %xmm203076; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm33077; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm123078; AVX512-NEXT:    vmovdqa 48(%rdi), %xmm73079; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]3080; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero3081; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]3082; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm43083; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]3084; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero3085; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3086; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm3, %ymm03087; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm183088; AVX512-NEXT:    vmovdqa (%rcx), %xmm33089; AVX512-NEXT:    vmovdqa 16(%rcx), %xmm53090; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm133091; AVX512-NEXT:    vmovdqa 48(%rcx), %xmm93092; AVX512-NEXT:    vmovdqa (%rdx), %xmm43093; AVX512-NEXT:    vmovdqa 16(%rdx), %xmm83094; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm143095; AVX512-NEXT:    vmovdqa 48(%rdx), %xmm103096; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7]3097; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]3098; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm5 = [0,0,1,1,2,2,3,3,16,16,17,17,18,18,19,19]3099; AVX512-NEXT:    vpermt2d %zmm15, %zmm5, %zmm83100; AVX512-NEXT:    movw $-21846, %ax # imm = 0xAAAA3101; AVX512-NEXT:    kmovw %eax, %k13102; AVX512-NEXT:    vmovdqa32 %zmm8, %zmm18 {%k1}3103; AVX512-NEXT:    vmovdqa 96(%rsi), %xmm83104; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm153105; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm15[4],xmm8[4],xmm15[5],xmm8[5],xmm15[6],xmm8[6],xmm15[7],xmm8[7]3106; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm16 = xmm0[0],zero,xmm0[1],zero3107; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3108; AVX512-NEXT:    vinserti32x4 $1, %xmm0, %ymm16, %ymm03109; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3]3110; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm15 = xmm8[0],zero,xmm8[1],zero3111; AVX512-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]3112; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm15, %ymm83113; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm8, %zmm83114; AVX512-NEXT:    vmovdqa 96(%rcx), %xmm03115; AVX512-NEXT:    vmovdqa 96(%rdx), %xmm153116; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7]3117; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]3118; AVX512-NEXT:    vpermt2d %zmm1, %zmm5, %zmm03119; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm8 {%k1}3120; AVX512-NEXT:    vmovdqa 112(%rsi), %xmm03121; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm13122; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3123; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm16 = xmm15[0],zero,xmm15[1],zero3124; AVX512-NEXT:    vpshufd {{.*#+}} xmm15 = xmm15[2,2,3,3]3125; AVX512-NEXT:    vinserti32x4 $1, %xmm15, %ymm16, %ymm153126; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3127; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero3128; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3129; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm03130; AVX512-NEXT:    vinserti64x4 $1, %ymm15, %zmm0, %zmm153131; AVX512-NEXT:    vmovdqa 112(%rcx), %xmm03132; AVX512-NEXT:    vmovdqa 112(%rdx), %xmm13133; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3134; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3135; AVX512-NEXT:    vpermt2d %zmm2, %zmm5, %zmm03136; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm15 {%k1}3137; AVX512-NEXT:    vmovdqa 64(%rsi), %xmm03138; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm13139; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3140; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm16 = xmm2[0],zero,xmm2[1],zero3141; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]3142; AVX512-NEXT:    vinserti32x4 $1, %xmm2, %ymm16, %ymm23143; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3144; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero3145; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3146; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm03147; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm163148; AVX512-NEXT:    vmovdqa 64(%rcx), %xmm03149; AVX512-NEXT:    vmovdqa 64(%rdx), %xmm13150; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3151; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3152; AVX512-NEXT:    vpermt2d %zmm2, %zmm5, %zmm03153; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm16 {%k1}3154; AVX512-NEXT:    vmovdqa 80(%rsi), %xmm03155; AVX512-NEXT:    vmovdqa 80(%rdi), %xmm13156; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3157; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm17 = xmm2[0],zero,xmm2[1],zero3158; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]3159; AVX512-NEXT:    vinserti32x4 $1, %xmm2, %ymm17, %ymm23160; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3161; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero3162; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3163; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm03164; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm173165; AVX512-NEXT:    vmovdqa 80(%rcx), %xmm03166; AVX512-NEXT:    vmovdqa 80(%rdx), %xmm13167; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3168; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3169; AVX512-NEXT:    vpermt2d %zmm2, %zmm5, %zmm03170; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm17 {%k1}3171; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]3172; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero3173; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3174; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm03175; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]3176; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero3177; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]3178; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm13179; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm03180; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]3181; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]3182; AVX512-NEXT:    vpermt2d %zmm1, %zmm5, %zmm23183; AVX512-NEXT:    vmovdqa32 %zmm2, %zmm0 {%k1}3184; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]3185; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero3186; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]3187; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm13188; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]3189; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero3190; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]3191; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm6, %ymm23192; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm13193; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]3194; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]3195; AVX512-NEXT:    vpermt2d %zmm2, %zmm5, %zmm63196; AVX512-NEXT:    vmovdqa32 %zmm6, %zmm1 {%k1}3197; AVX512-NEXT:    vmovdqa64 %xmm19, %xmm73198; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm93199; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm9[4],xmm7[4],xmm9[5],xmm7[5],xmm9[6],xmm7[6],xmm9[7],xmm7[7]3200; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero3201; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]3202; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm6, %ymm23203; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]3204; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero3205; AVX512-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]3206; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm7, %ymm63207; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm6, %zmm23208; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]3209; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]3210; AVX512-NEXT:    vpermt2d %zmm6, %zmm5, %zmm33211; AVX512-NEXT:    vmovdqa32 %zmm3, %zmm2 {%k1}3212; AVX512-NEXT:    vmovdqa64 %zmm2, (%r8)3213; AVX512-NEXT:    vmovdqa64 %zmm1, 192(%r8)3214; AVX512-NEXT:    vmovdqa64 %zmm0, 128(%r8)3215; AVX512-NEXT:    vmovdqa64 %zmm17, 320(%r8)3216; AVX512-NEXT:    vmovdqa64 %zmm16, 256(%r8)3217; AVX512-NEXT:    vmovdqa64 %zmm15, 448(%r8)3218; AVX512-NEXT:    vmovdqa64 %zmm8, 384(%r8)3219; AVX512-NEXT:    vmovdqa64 %zmm18, 64(%r8)3220; AVX512-NEXT:    vzeroupper3221; AVX512-NEXT:    retq3222;3223; AVX512-FCP-LABEL: store_i16_stride4_vf64:3224; AVX512-FCP:       # %bb.0:3225; AVX512-FCP-NEXT:    vmovdqa64 (%rsi), %xmm193226; AVX512-FCP-NEXT:    vmovdqa 16(%rsi), %xmm03227; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm113228; AVX512-FCP-NEXT:    vmovdqa 48(%rsi), %xmm63229; AVX512-FCP-NEXT:    vmovdqa64 (%rdi), %xmm203230; AVX512-FCP-NEXT:    vmovdqa 16(%rdi), %xmm33231; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm123232; AVX512-FCP-NEXT:    vmovdqa 48(%rdi), %xmm73233; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]3234; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero3235; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]3236; AVX512-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm43237; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]3238; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero3239; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3240; AVX512-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm3, %ymm03241; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm183242; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm33243; AVX512-FCP-NEXT:    vmovdqa 16(%rcx), %xmm53244; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm133245; AVX512-FCP-NEXT:    vmovdqa 48(%rcx), %xmm93246; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm43247; AVX512-FCP-NEXT:    vmovdqa 16(%rdx), %xmm83248; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm143249; AVX512-FCP-NEXT:    vmovdqa 48(%rdx), %xmm103250; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7]3251; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]3252; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm5 = [0,0,1,1,2,2,3,3,16,16,17,17,18,18,19,19]3253; AVX512-FCP-NEXT:    vpermt2d %zmm15, %zmm5, %zmm83254; AVX512-FCP-NEXT:    movw $-21846, %ax # imm = 0xAAAA3255; AVX512-FCP-NEXT:    kmovw %eax, %k13256; AVX512-FCP-NEXT:    vmovdqa32 %zmm8, %zmm18 {%k1}3257; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %xmm83258; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm153259; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm15[4],xmm8[4],xmm15[5],xmm8[5],xmm15[6],xmm8[6],xmm15[7],xmm8[7]3260; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm16 = xmm0[0],zero,xmm0[1],zero3261; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3262; AVX512-FCP-NEXT:    vinserti32x4 $1, %xmm0, %ymm16, %ymm03263; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3]3264; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm15 = xmm8[0],zero,xmm8[1],zero3265; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]3266; AVX512-FCP-NEXT:    vinserti128 $1, %xmm8, %ymm15, %ymm83267; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm8, %zmm83268; AVX512-FCP-NEXT:    vmovdqa 96(%rcx), %xmm03269; AVX512-FCP-NEXT:    vmovdqa 96(%rdx), %xmm153270; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7]3271; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]3272; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm5, %zmm03273; AVX512-FCP-NEXT:    vmovdqa32 %zmm0, %zmm8 {%k1}3274; AVX512-FCP-NEXT:    vmovdqa 112(%rsi), %xmm03275; AVX512-FCP-NEXT:    vmovdqa 112(%rdi), %xmm13276; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3277; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm16 = xmm15[0],zero,xmm15[1],zero3278; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm15 = xmm15[2,2,3,3]3279; AVX512-FCP-NEXT:    vinserti32x4 $1, %xmm15, %ymm16, %ymm153280; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3281; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero3282; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3283; AVX512-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm03284; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm15, %zmm0, %zmm153285; AVX512-FCP-NEXT:    vmovdqa 112(%rcx), %xmm03286; AVX512-FCP-NEXT:    vmovdqa 112(%rdx), %xmm13287; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3288; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3289; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm5, %zmm03290; AVX512-FCP-NEXT:    vmovdqa32 %zmm0, %zmm15 {%k1}3291; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %xmm03292; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm13293; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3294; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm16 = xmm2[0],zero,xmm2[1],zero3295; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]3296; AVX512-FCP-NEXT:    vinserti32x4 $1, %xmm2, %ymm16, %ymm23297; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3298; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero3299; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3300; AVX512-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm03301; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm163302; AVX512-FCP-NEXT:    vmovdqa 64(%rcx), %xmm03303; AVX512-FCP-NEXT:    vmovdqa 64(%rdx), %xmm13304; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3305; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3306; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm5, %zmm03307; AVX512-FCP-NEXT:    vmovdqa32 %zmm0, %zmm16 {%k1}3308; AVX512-FCP-NEXT:    vmovdqa 80(%rsi), %xmm03309; AVX512-FCP-NEXT:    vmovdqa 80(%rdi), %xmm13310; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3311; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm17 = xmm2[0],zero,xmm2[1],zero3312; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]3313; AVX512-FCP-NEXT:    vinserti32x4 $1, %xmm2, %ymm17, %ymm23314; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3315; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero3316; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3317; AVX512-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm03318; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm173319; AVX512-FCP-NEXT:    vmovdqa 80(%rcx), %xmm03320; AVX512-FCP-NEXT:    vmovdqa 80(%rdx), %xmm13321; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3322; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3323; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm5, %zmm03324; AVX512-FCP-NEXT:    vmovdqa32 %zmm0, %zmm17 {%k1}3325; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]3326; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero3327; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3328; AVX512-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm03329; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]3330; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero3331; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]3332; AVX512-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm13333; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm03334; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]3335; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]3336; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm5, %zmm23337; AVX512-FCP-NEXT:    vmovdqa32 %zmm2, %zmm0 {%k1}3338; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]3339; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero3340; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]3341; AVX512-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm13342; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]3343; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero3344; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]3345; AVX512-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm6, %ymm23346; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm13347; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]3348; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]3349; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm5, %zmm63350; AVX512-FCP-NEXT:    vmovdqa32 %zmm6, %zmm1 {%k1}3351; AVX512-FCP-NEXT:    vmovdqa64 %xmm19, %xmm73352; AVX512-FCP-NEXT:    vmovdqa64 %xmm20, %xmm93353; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm9[4],xmm7[4],xmm9[5],xmm7[5],xmm9[6],xmm7[6],xmm9[7],xmm7[7]3354; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero3355; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]3356; AVX512-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm6, %ymm23357; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]3358; AVX512-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero3359; AVX512-FCP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]3360; AVX512-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm7, %ymm63361; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm6, %zmm23362; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]3363; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]3364; AVX512-FCP-NEXT:    vpermt2d %zmm6, %zmm5, %zmm33365; AVX512-FCP-NEXT:    vmovdqa32 %zmm3, %zmm2 {%k1}3366; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, (%r8)3367; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 192(%r8)3368; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 128(%r8)3369; AVX512-FCP-NEXT:    vmovdqa64 %zmm17, 320(%r8)3370; AVX512-FCP-NEXT:    vmovdqa64 %zmm16, 256(%r8)3371; AVX512-FCP-NEXT:    vmovdqa64 %zmm15, 448(%r8)3372; AVX512-FCP-NEXT:    vmovdqa64 %zmm8, 384(%r8)3373; AVX512-FCP-NEXT:    vmovdqa64 %zmm18, 64(%r8)3374; AVX512-FCP-NEXT:    vzeroupper3375; AVX512-FCP-NEXT:    retq3376;3377; AVX512DQ-LABEL: store_i16_stride4_vf64:3378; AVX512DQ:       # %bb.0:3379; AVX512DQ-NEXT:    vmovdqa64 (%rsi), %xmm193380; AVX512DQ-NEXT:    vmovdqa 16(%rsi), %xmm03381; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm113382; AVX512DQ-NEXT:    vmovdqa 48(%rsi), %xmm63383; AVX512DQ-NEXT:    vmovdqa64 (%rdi), %xmm203384; AVX512DQ-NEXT:    vmovdqa 16(%rdi), %xmm33385; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm123386; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm73387; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]3388; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero3389; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]3390; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm43391; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]3392; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero3393; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3394; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm3, %ymm03395; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm183396; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm33397; AVX512DQ-NEXT:    vmovdqa 16(%rcx), %xmm53398; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm133399; AVX512DQ-NEXT:    vmovdqa 48(%rcx), %xmm93400; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm43401; AVX512DQ-NEXT:    vmovdqa 16(%rdx), %xmm83402; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm143403; AVX512DQ-NEXT:    vmovdqa 48(%rdx), %xmm103404; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7]3405; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]3406; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm5 = [0,0,1,1,2,2,3,3,16,16,17,17,18,18,19,19]3407; AVX512DQ-NEXT:    vpermt2d %zmm15, %zmm5, %zmm83408; AVX512DQ-NEXT:    movw $-21846, %ax # imm = 0xAAAA3409; AVX512DQ-NEXT:    kmovw %eax, %k13410; AVX512DQ-NEXT:    vmovdqa32 %zmm8, %zmm18 {%k1}3411; AVX512DQ-NEXT:    vmovdqa 96(%rsi), %xmm83412; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm153413; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm15[4],xmm8[4],xmm15[5],xmm8[5],xmm15[6],xmm8[6],xmm15[7],xmm8[7]3414; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm16 = xmm0[0],zero,xmm0[1],zero3415; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3416; AVX512DQ-NEXT:    vinserti32x4 $1, %xmm0, %ymm16, %ymm03417; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3]3418; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm15 = xmm8[0],zero,xmm8[1],zero3419; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]3420; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm15, %ymm83421; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm8, %zmm83422; AVX512DQ-NEXT:    vmovdqa 96(%rcx), %xmm03423; AVX512DQ-NEXT:    vmovdqa 96(%rdx), %xmm153424; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7]3425; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]3426; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm5, %zmm03427; AVX512DQ-NEXT:    vmovdqa32 %zmm0, %zmm8 {%k1}3428; AVX512DQ-NEXT:    vmovdqa 112(%rsi), %xmm03429; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm13430; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3431; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm16 = xmm15[0],zero,xmm15[1],zero3432; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm15 = xmm15[2,2,3,3]3433; AVX512DQ-NEXT:    vinserti32x4 $1, %xmm15, %ymm16, %ymm153434; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3435; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero3436; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3437; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm03438; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm15, %zmm0, %zmm153439; AVX512DQ-NEXT:    vmovdqa 112(%rcx), %xmm03440; AVX512DQ-NEXT:    vmovdqa 112(%rdx), %xmm13441; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3442; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3443; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm5, %zmm03444; AVX512DQ-NEXT:    vmovdqa32 %zmm0, %zmm15 {%k1}3445; AVX512DQ-NEXT:    vmovdqa 64(%rsi), %xmm03446; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm13447; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3448; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm16 = xmm2[0],zero,xmm2[1],zero3449; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]3450; AVX512DQ-NEXT:    vinserti32x4 $1, %xmm2, %ymm16, %ymm23451; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3452; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero3453; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3454; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm03455; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm163456; AVX512DQ-NEXT:    vmovdqa 64(%rcx), %xmm03457; AVX512DQ-NEXT:    vmovdqa 64(%rdx), %xmm13458; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3459; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3460; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm5, %zmm03461; AVX512DQ-NEXT:    vmovdqa32 %zmm0, %zmm16 {%k1}3462; AVX512DQ-NEXT:    vmovdqa 80(%rsi), %xmm03463; AVX512DQ-NEXT:    vmovdqa 80(%rdi), %xmm13464; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3465; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm17 = xmm2[0],zero,xmm2[1],zero3466; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]3467; AVX512DQ-NEXT:    vinserti32x4 $1, %xmm2, %ymm17, %ymm23468; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3469; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero3470; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3471; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm03472; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm173473; AVX512DQ-NEXT:    vmovdqa 80(%rcx), %xmm03474; AVX512DQ-NEXT:    vmovdqa 80(%rdx), %xmm13475; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3476; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3477; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm5, %zmm03478; AVX512DQ-NEXT:    vmovdqa32 %zmm0, %zmm17 {%k1}3479; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]3480; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero3481; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3482; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm03483; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]3484; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero3485; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]3486; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm13487; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm03488; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]3489; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]3490; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm5, %zmm23491; AVX512DQ-NEXT:    vmovdqa32 %zmm2, %zmm0 {%k1}3492; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]3493; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero3494; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]3495; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm13496; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]3497; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero3498; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]3499; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm6, %ymm23500; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm13501; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]3502; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]3503; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm5, %zmm63504; AVX512DQ-NEXT:    vmovdqa32 %zmm6, %zmm1 {%k1}3505; AVX512DQ-NEXT:    vmovdqa64 %xmm19, %xmm73506; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm93507; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm9[4],xmm7[4],xmm9[5],xmm7[5],xmm9[6],xmm7[6],xmm9[7],xmm7[7]3508; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero3509; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]3510; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm6, %ymm23511; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]3512; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero3513; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]3514; AVX512DQ-NEXT:    vinserti128 $1, %xmm6, %ymm7, %ymm63515; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm6, %zmm23516; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]3517; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]3518; AVX512DQ-NEXT:    vpermt2d %zmm6, %zmm5, %zmm33519; AVX512DQ-NEXT:    vmovdqa32 %zmm3, %zmm2 {%k1}3520; AVX512DQ-NEXT:    vmovdqa64 %zmm2, (%r8)3521; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 192(%r8)3522; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 128(%r8)3523; AVX512DQ-NEXT:    vmovdqa64 %zmm17, 320(%r8)3524; AVX512DQ-NEXT:    vmovdqa64 %zmm16, 256(%r8)3525; AVX512DQ-NEXT:    vmovdqa64 %zmm15, 448(%r8)3526; AVX512DQ-NEXT:    vmovdqa64 %zmm8, 384(%r8)3527; AVX512DQ-NEXT:    vmovdqa64 %zmm18, 64(%r8)3528; AVX512DQ-NEXT:    vzeroupper3529; AVX512DQ-NEXT:    retq3530;3531; AVX512DQ-FCP-LABEL: store_i16_stride4_vf64:3532; AVX512DQ-FCP:       # %bb.0:3533; AVX512DQ-FCP-NEXT:    vmovdqa64 (%rsi), %xmm193534; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rsi), %xmm03535; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm113536; AVX512DQ-FCP-NEXT:    vmovdqa 48(%rsi), %xmm63537; AVX512DQ-FCP-NEXT:    vmovdqa64 (%rdi), %xmm203538; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rdi), %xmm33539; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm123540; AVX512DQ-FCP-NEXT:    vmovdqa 48(%rdi), %xmm73541; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]3542; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero3543; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,3,3]3544; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm5, %ymm43545; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]3546; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero3547; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3548; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm3, %ymm03549; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm183550; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm33551; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rcx), %xmm53552; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm133553; AVX512DQ-FCP-NEXT:    vmovdqa 48(%rcx), %xmm93554; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm43555; AVX512DQ-FCP-NEXT:    vmovdqa 16(%rdx), %xmm83556; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm143557; AVX512DQ-FCP-NEXT:    vmovdqa 48(%rdx), %xmm103558; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7]3559; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]3560; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm5 = [0,0,1,1,2,2,3,3,16,16,17,17,18,18,19,19]3561; AVX512DQ-FCP-NEXT:    vpermt2d %zmm15, %zmm5, %zmm83562; AVX512DQ-FCP-NEXT:    movw $-21846, %ax # imm = 0xAAAA3563; AVX512DQ-FCP-NEXT:    kmovw %eax, %k13564; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm8, %zmm18 {%k1}3565; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %xmm83566; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm153567; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm15[4],xmm8[4],xmm15[5],xmm8[5],xmm15[6],xmm8[6],xmm15[7],xmm8[7]3568; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm16 = xmm0[0],zero,xmm0[1],zero3569; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3570; AVX512DQ-FCP-NEXT:    vinserti32x4 $1, %xmm0, %ymm16, %ymm03571; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3]3572; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm15 = xmm8[0],zero,xmm8[1],zero3573; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm8 = xmm8[2,2,3,3]3574; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm8, %ymm15, %ymm83575; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm8, %zmm83576; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rcx), %xmm03577; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdx), %xmm153578; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7]3579; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]3580; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm5, %zmm03581; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm0, %zmm8 {%k1}3582; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rsi), %xmm03583; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rdi), %xmm13584; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3585; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm16 = xmm15[0],zero,xmm15[1],zero3586; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm15 = xmm15[2,2,3,3]3587; AVX512DQ-FCP-NEXT:    vinserti32x4 $1, %xmm15, %ymm16, %ymm153588; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3589; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero3590; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3591; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm03592; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm15, %zmm0, %zmm153593; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rcx), %xmm03594; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rdx), %xmm13595; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3596; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3597; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm5, %zmm03598; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm0, %zmm15 {%k1}3599; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rsi), %xmm03600; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %xmm13601; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3602; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm16 = xmm2[0],zero,xmm2[1],zero3603; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]3604; AVX512DQ-FCP-NEXT:    vinserti32x4 $1, %xmm2, %ymm16, %ymm23605; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3606; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero3607; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3608; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm03609; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm163610; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rcx), %xmm03611; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdx), %xmm13612; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3613; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3614; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm5, %zmm03615; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm0, %zmm16 {%k1}3616; AVX512DQ-FCP-NEXT:    vmovdqa 80(%rsi), %xmm03617; AVX512DQ-FCP-NEXT:    vmovdqa 80(%rdi), %xmm13618; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3619; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm17 = xmm2[0],zero,xmm2[1],zero3620; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]3621; AVX512DQ-FCP-NEXT:    vinserti32x4 $1, %xmm2, %ymm17, %ymm23622; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3623; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero3624; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3625; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm03626; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm173627; AVX512DQ-FCP-NEXT:    vmovdqa 80(%rcx), %xmm03628; AVX512DQ-FCP-NEXT:    vmovdqa 80(%rdx), %xmm13629; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3630; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3631; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm5, %zmm03632; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm0, %zmm17 {%k1}3633; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]3634; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero3635; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]3636; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm03637; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]3638; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero3639; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]3640; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm13641; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm03642; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]3643; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]3644; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm5, %zmm23645; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm2, %zmm0 {%k1}3646; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]3647; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero3648; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]3649; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm13650; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]3651; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero3652; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]3653; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm6, %ymm23654; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm13655; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]3656; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]3657; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm5, %zmm63658; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm6, %zmm1 {%k1}3659; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm19, %xmm73660; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm20, %xmm93661; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm9[4],xmm7[4],xmm9[5],xmm7[5],xmm9[6],xmm7[6],xmm9[7],xmm7[7]3662; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm2[0],zero,xmm2[1],zero3663; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]3664; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm6, %ymm23665; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]3666; AVX512DQ-FCP-NEXT:    vpmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero3667; AVX512DQ-FCP-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]3668; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm7, %ymm63669; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm6, %zmm23670; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]3671; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]3672; AVX512DQ-FCP-NEXT:    vpermt2d %zmm6, %zmm5, %zmm33673; AVX512DQ-FCP-NEXT:    vmovdqa32 %zmm3, %zmm2 {%k1}3674; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, (%r8)3675; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 192(%r8)3676; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 128(%r8)3677; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm17, 320(%r8)3678; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm16, 256(%r8)3679; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm15, 448(%r8)3680; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm8, 384(%r8)3681; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm18, 64(%r8)3682; AVX512DQ-FCP-NEXT:    vzeroupper3683; AVX512DQ-FCP-NEXT:    retq3684;3685; AVX512BW-LABEL: store_i16_stride4_vf64:3686; AVX512BW:       # %bb.0:3687; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm03688; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm13689; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm23690; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %zmm33691; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm43692; AVX512BW-NEXT:    vmovdqa64 64(%rdx), %zmm53693; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm63694; AVX512BW-NEXT:    vmovdqa64 64(%rcx), %zmm73695; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm8 = [0,0,8,40,0,0,9,41,0,0,10,42,0,0,11,43,0,0,12,44,0,0,13,45,0,0,14,46,0,0,15,47]3696; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm93697; AVX512BW-NEXT:    vpermt2w %zmm6, %zmm8, %zmm93698; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [8,40,0,0,9,41,0,0,10,42,0,0,11,43,0,0,12,44,0,0,13,45,0,0,14,46,0,0,15,47,0,0]3699; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm113700; AVX512BW-NEXT:    vpermt2w %zmm2, %zmm10, %zmm113701; AVX512BW-NEXT:    movw $-21846, %ax # imm = 0xAAAA3702; AVX512BW-NEXT:    kmovd %eax, %k13703; AVX512BW-NEXT:    vmovdqa32 %zmm9, %zmm11 {%k1}3704; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39]3705; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm123706; AVX512BW-NEXT:    vpermt2w %zmm6, %zmm9, %zmm123707; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39,0,0]3708; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm143709; AVX512BW-NEXT:    vpermt2w %zmm2, %zmm13, %zmm143710; AVX512BW-NEXT:    vmovdqa32 %zmm12, %zmm14 {%k1}3711; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,24,56,0,0,25,57,0,0,26,58,0,0,27,59,0,0,28,60,0,0,29,61,0,0,30,62,0,0,31,63]3712; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm153713; AVX512BW-NEXT:    vpermt2w %zmm6, %zmm12, %zmm153714; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm16 = [24,56,0,0,25,57,0,0,26,58,0,0,27,59,0,0,28,60,0,0,29,61,0,0,30,62,0,0,31,63,0,0]3715; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm173716; AVX512BW-NEXT:    vpermt2w %zmm2, %zmm16, %zmm173717; AVX512BW-NEXT:    vmovdqa32 %zmm15, %zmm17 {%k1}3718; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm15 = [0,0,16,48,0,0,17,49,0,0,18,50,0,0,19,51,0,0,20,52,0,0,21,53,0,0,22,54,0,0,23,55]3719; AVX512BW-NEXT:    vpermt2w %zmm6, %zmm15, %zmm43720; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [16,48,0,0,17,49,0,0,18,50,0,0,19,51,0,0,20,52,0,0,21,53,0,0,22,54,0,0,23,55,0,0]3721; AVX512BW-NEXT:    vpermt2w %zmm2, %zmm6, %zmm03722; AVX512BW-NEXT:    vmovdqa32 %zmm4, %zmm0 {%k1}3723; AVX512BW-NEXT:    vpermi2w %zmm7, %zmm5, %zmm83724; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm1, %zmm103725; AVX512BW-NEXT:    vmovdqa32 %zmm8, %zmm10 {%k1}3726; AVX512BW-NEXT:    vpermi2w %zmm7, %zmm5, %zmm93727; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm1, %zmm133728; AVX512BW-NEXT:    vmovdqa32 %zmm9, %zmm13 {%k1}3729; AVX512BW-NEXT:    vpermi2w %zmm7, %zmm5, %zmm123730; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm1, %zmm163731; AVX512BW-NEXT:    vmovdqa32 %zmm12, %zmm16 {%k1}3732; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm15, %zmm53733; AVX512BW-NEXT:    vpermt2w %zmm3, %zmm6, %zmm13734; AVX512BW-NEXT:    vmovdqa32 %zmm5, %zmm1 {%k1}3735; AVX512BW-NEXT:    vmovdqa64 %zmm1, 384(%r8)3736; AVX512BW-NEXT:    vmovdqa64 %zmm16, 448(%r8)3737; AVX512BW-NEXT:    vmovdqa64 %zmm13, 256(%r8)3738; AVX512BW-NEXT:    vmovdqa64 %zmm10, 320(%r8)3739; AVX512BW-NEXT:    vmovdqa64 %zmm0, 128(%r8)3740; AVX512BW-NEXT:    vmovdqa64 %zmm17, 192(%r8)3741; AVX512BW-NEXT:    vmovdqa64 %zmm14, (%r8)3742; AVX512BW-NEXT:    vmovdqa64 %zmm11, 64(%r8)3743; AVX512BW-NEXT:    vzeroupper3744; AVX512BW-NEXT:    retq3745;3746; AVX512BW-FCP-LABEL: store_i16_stride4_vf64:3747; AVX512BW-FCP:       # %bb.0:3748; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdi), %zmm03749; AVX512BW-FCP-NEXT:    vmovdqa64 64(%rdi), %zmm13750; AVX512BW-FCP-NEXT:    vmovdqa64 (%rsi), %zmm23751; AVX512BW-FCP-NEXT:    vmovdqa64 64(%rsi), %zmm33752; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdx), %zmm43753; AVX512BW-FCP-NEXT:    vmovdqa64 64(%rdx), %zmm53754; AVX512BW-FCP-NEXT:    vmovdqa64 (%rcx), %zmm63755; AVX512BW-FCP-NEXT:    vmovdqa64 64(%rcx), %zmm73756; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm8 = [0,0,8,40,0,0,9,41,0,0,10,42,0,0,11,43,0,0,12,44,0,0,13,45,0,0,14,46,0,0,15,47]3757; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm93758; AVX512BW-FCP-NEXT:    vpermt2w %zmm6, %zmm8, %zmm93759; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [8,40,0,0,9,41,0,0,10,42,0,0,11,43,0,0,12,44,0,0,13,45,0,0,14,46,0,0,15,47,0,0]3760; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm113761; AVX512BW-FCP-NEXT:    vpermt2w %zmm2, %zmm10, %zmm113762; AVX512BW-FCP-NEXT:    movw $-21846, %ax # imm = 0xAAAA3763; AVX512BW-FCP-NEXT:    kmovd %eax, %k13764; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm9, %zmm11 {%k1}3765; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39]3766; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm123767; AVX512BW-FCP-NEXT:    vpermt2w %zmm6, %zmm9, %zmm123768; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39,0,0]3769; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm143770; AVX512BW-FCP-NEXT:    vpermt2w %zmm2, %zmm13, %zmm143771; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm14 {%k1}3772; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,24,56,0,0,25,57,0,0,26,58,0,0,27,59,0,0,28,60,0,0,29,61,0,0,30,62,0,0,31,63]3773; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm153774; AVX512BW-FCP-NEXT:    vpermt2w %zmm6, %zmm12, %zmm153775; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm16 = [24,56,0,0,25,57,0,0,26,58,0,0,27,59,0,0,28,60,0,0,29,61,0,0,30,62,0,0,31,63,0,0]3776; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm173777; AVX512BW-FCP-NEXT:    vpermt2w %zmm2, %zmm16, %zmm173778; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm15, %zmm17 {%k1}3779; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm15 = [0,0,16,48,0,0,17,49,0,0,18,50,0,0,19,51,0,0,20,52,0,0,21,53,0,0,22,54,0,0,23,55]3780; AVX512BW-FCP-NEXT:    vpermt2w %zmm6, %zmm15, %zmm43781; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [16,48,0,0,17,49,0,0,18,50,0,0,19,51,0,0,20,52,0,0,21,53,0,0,22,54,0,0,23,55,0,0]3782; AVX512BW-FCP-NEXT:    vpermt2w %zmm2, %zmm6, %zmm03783; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm4, %zmm0 {%k1}3784; AVX512BW-FCP-NEXT:    vpermi2w %zmm7, %zmm5, %zmm83785; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm1, %zmm103786; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm8, %zmm10 {%k1}3787; AVX512BW-FCP-NEXT:    vpermi2w %zmm7, %zmm5, %zmm93788; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm1, %zmm133789; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm9, %zmm13 {%k1}3790; AVX512BW-FCP-NEXT:    vpermi2w %zmm7, %zmm5, %zmm123791; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm1, %zmm163792; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm16 {%k1}3793; AVX512BW-FCP-NEXT:    vpermt2w %zmm7, %zmm15, %zmm53794; AVX512BW-FCP-NEXT:    vpermt2w %zmm3, %zmm6, %zmm13795; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm5, %zmm1 {%k1}3796; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm1, 384(%r8)3797; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm16, 448(%r8)3798; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm13, 256(%r8)3799; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm10, 320(%r8)3800; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, 128(%r8)3801; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm17, 192(%r8)3802; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm14, (%r8)3803; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm11, 64(%r8)3804; AVX512BW-FCP-NEXT:    vzeroupper3805; AVX512BW-FCP-NEXT:    retq3806;3807; AVX512DQ-BW-LABEL: store_i16_stride4_vf64:3808; AVX512DQ-BW:       # %bb.0:3809; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdi), %zmm03810; AVX512DQ-BW-NEXT:    vmovdqa64 64(%rdi), %zmm13811; AVX512DQ-BW-NEXT:    vmovdqa64 (%rsi), %zmm23812; AVX512DQ-BW-NEXT:    vmovdqa64 64(%rsi), %zmm33813; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdx), %zmm43814; AVX512DQ-BW-NEXT:    vmovdqa64 64(%rdx), %zmm53815; AVX512DQ-BW-NEXT:    vmovdqa64 (%rcx), %zmm63816; AVX512DQ-BW-NEXT:    vmovdqa64 64(%rcx), %zmm73817; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm8 = [0,0,8,40,0,0,9,41,0,0,10,42,0,0,11,43,0,0,12,44,0,0,13,45,0,0,14,46,0,0,15,47]3818; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, %zmm93819; AVX512DQ-BW-NEXT:    vpermt2w %zmm6, %zmm8, %zmm93820; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [8,40,0,0,9,41,0,0,10,42,0,0,11,43,0,0,12,44,0,0,13,45,0,0,14,46,0,0,15,47,0,0]3821; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, %zmm113822; AVX512DQ-BW-NEXT:    vpermt2w %zmm2, %zmm10, %zmm113823; AVX512DQ-BW-NEXT:    movw $-21846, %ax # imm = 0xAAAA3824; AVX512DQ-BW-NEXT:    kmovd %eax, %k13825; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm9, %zmm11 {%k1}3826; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39]3827; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, %zmm123828; AVX512DQ-BW-NEXT:    vpermt2w %zmm6, %zmm9, %zmm123829; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39,0,0]3830; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, %zmm143831; AVX512DQ-BW-NEXT:    vpermt2w %zmm2, %zmm13, %zmm143832; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm12, %zmm14 {%k1}3833; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,24,56,0,0,25,57,0,0,26,58,0,0,27,59,0,0,28,60,0,0,29,61,0,0,30,62,0,0,31,63]3834; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, %zmm153835; AVX512DQ-BW-NEXT:    vpermt2w %zmm6, %zmm12, %zmm153836; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm16 = [24,56,0,0,25,57,0,0,26,58,0,0,27,59,0,0,28,60,0,0,29,61,0,0,30,62,0,0,31,63,0,0]3837; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, %zmm173838; AVX512DQ-BW-NEXT:    vpermt2w %zmm2, %zmm16, %zmm173839; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm15, %zmm17 {%k1}3840; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm15 = [0,0,16,48,0,0,17,49,0,0,18,50,0,0,19,51,0,0,20,52,0,0,21,53,0,0,22,54,0,0,23,55]3841; AVX512DQ-BW-NEXT:    vpermt2w %zmm6, %zmm15, %zmm43842; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [16,48,0,0,17,49,0,0,18,50,0,0,19,51,0,0,20,52,0,0,21,53,0,0,22,54,0,0,23,55,0,0]3843; AVX512DQ-BW-NEXT:    vpermt2w %zmm2, %zmm6, %zmm03844; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm4, %zmm0 {%k1}3845; AVX512DQ-BW-NEXT:    vpermi2w %zmm7, %zmm5, %zmm83846; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm1, %zmm103847; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm8, %zmm10 {%k1}3848; AVX512DQ-BW-NEXT:    vpermi2w %zmm7, %zmm5, %zmm93849; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm1, %zmm133850; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm9, %zmm13 {%k1}3851; AVX512DQ-BW-NEXT:    vpermi2w %zmm7, %zmm5, %zmm123852; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm1, %zmm163853; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm12, %zmm16 {%k1}3854; AVX512DQ-BW-NEXT:    vpermt2w %zmm7, %zmm15, %zmm53855; AVX512DQ-BW-NEXT:    vpermt2w %zmm3, %zmm6, %zmm13856; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm5, %zmm1 {%k1}3857; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm1, 384(%r8)3858; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm16, 448(%r8)3859; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm13, 256(%r8)3860; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm10, 320(%r8)3861; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, 128(%r8)3862; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm17, 192(%r8)3863; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm14, (%r8)3864; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm11, 64(%r8)3865; AVX512DQ-BW-NEXT:    vzeroupper3866; AVX512DQ-BW-NEXT:    retq3867;3868; AVX512DQ-BW-FCP-LABEL: store_i16_stride4_vf64:3869; AVX512DQ-BW-FCP:       # %bb.0:3870; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdi), %zmm03871; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 64(%rdi), %zmm13872; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rsi), %zmm23873; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 64(%rsi), %zmm33874; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdx), %zmm43875; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 64(%rdx), %zmm53876; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rcx), %zmm63877; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 64(%rcx), %zmm73878; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm8 = [0,0,8,40,0,0,9,41,0,0,10,42,0,0,11,43,0,0,12,44,0,0,13,45,0,0,14,46,0,0,15,47]3879; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm93880; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm6, %zmm8, %zmm93881; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [8,40,0,0,9,41,0,0,10,42,0,0,11,43,0,0,12,44,0,0,13,45,0,0,14,46,0,0,15,47,0,0]3882; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm113883; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm2, %zmm10, %zmm113884; AVX512DQ-BW-FCP-NEXT:    movw $-21846, %ax # imm = 0xAAAA3885; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k13886; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm9, %zmm11 {%k1}3887; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39]3888; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm123889; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm6, %zmm9, %zmm123890; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,32,0,0,1,33,0,0,2,34,0,0,3,35,0,0,4,36,0,0,5,37,0,0,6,38,0,0,7,39,0,0]3891; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm143892; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm2, %zmm13, %zmm143893; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm14 {%k1}3894; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,24,56,0,0,25,57,0,0,26,58,0,0,27,59,0,0,28,60,0,0,29,61,0,0,30,62,0,0,31,63]3895; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm153896; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm6, %zmm12, %zmm153897; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm16 = [24,56,0,0,25,57,0,0,26,58,0,0,27,59,0,0,28,60,0,0,29,61,0,0,30,62,0,0,31,63,0,0]3898; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm173899; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm2, %zmm16, %zmm173900; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm15, %zmm17 {%k1}3901; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm15 = [0,0,16,48,0,0,17,49,0,0,18,50,0,0,19,51,0,0,20,52,0,0,21,53,0,0,22,54,0,0,23,55]3902; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm6, %zmm15, %zmm43903; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [16,48,0,0,17,49,0,0,18,50,0,0,19,51,0,0,20,52,0,0,21,53,0,0,22,54,0,0,23,55,0,0]3904; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm2, %zmm6, %zmm03905; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm4, %zmm0 {%k1}3906; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm7, %zmm5, %zmm83907; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm1, %zmm103908; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm8, %zmm10 {%k1}3909; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm7, %zmm5, %zmm93910; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm1, %zmm133911; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm9, %zmm13 {%k1}3912; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm7, %zmm5, %zmm123913; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm1, %zmm163914; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm16 {%k1}3915; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm7, %zmm15, %zmm53916; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm3, %zmm6, %zmm13917; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm5, %zmm1 {%k1}3918; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm1, 384(%r8)3919; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm16, 448(%r8)3920; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm13, 256(%r8)3921; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm10, 320(%r8)3922; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, 128(%r8)3923; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm17, 192(%r8)3924; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm14, (%r8)3925; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm11, 64(%r8)3926; AVX512DQ-BW-FCP-NEXT:    vzeroupper3927; AVX512DQ-BW-FCP-NEXT:    retq3928  %in.vec0 = load <64 x i16>, ptr %in.vecptr0, align 643929  %in.vec1 = load <64 x i16>, ptr %in.vecptr1, align 643930  %in.vec2 = load <64 x i16>, ptr %in.vecptr2, align 643931  %in.vec3 = load <64 x i16>, ptr %in.vecptr3, align 643932  %1 = shufflevector <64 x i16> %in.vec0, <64 x i16> %in.vec1, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>3933  %2 = shufflevector <64 x i16> %in.vec2, <64 x i16> %in.vec3, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>3934  %3 = shufflevector <128 x i16> %1, <128 x i16> %2, <256 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127, i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143, i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151, i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159, i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191, i32 192, i32 193, i32 194, i32 195, i32 196, i32 197, i32 198, i32 199, i32 200, i32 201, i32 202, i32 203, i32 204, i32 205, i32 206, i32 207, i32 208, i32 209, i32 210, i32 211, i32 212, i32 213, i32 214, i32 215, i32 216, i32 217, i32 218, i32 219, i32 220, i32 221, i32 222, i32 223, i32 224, i32 225, i32 226, i32 227, i32 228, i32 229, i32 230, i32 231, i32 232, i32 233, i32 234, i32 235, i32 236, i32 237, i32 238, i32 239, i32 240, i32 241, i32 242, i32 243, i32 244, i32 245, i32 246, i32 247, i32 248, i32 249, i32 250, i32 251, i32 252, i32 253, i32 254, i32 255>3935  %interleaved.vec = shufflevector <256 x i16> %3, <256 x i16> poison, <256 x i32> <i32 0, i32 64, i32 128, i32 192, i32 1, i32 65, i32 129, i32 193, i32 2, i32 66, i32 130, i32 194, i32 3, i32 67, i32 131, i32 195, i32 4, i32 68, i32 132, i32 196, i32 5, i32 69, i32 133, i32 197, i32 6, i32 70, i32 134, i32 198, i32 7, i32 71, i32 135, i32 199, i32 8, i32 72, i32 136, i32 200, i32 9, i32 73, i32 137, i32 201, i32 10, i32 74, i32 138, i32 202, i32 11, i32 75, i32 139, i32 203, i32 12, i32 76, i32 140, i32 204, i32 13, i32 77, i32 141, i32 205, i32 14, i32 78, i32 142, i32 206, i32 15, i32 79, i32 143, i32 207, i32 16, i32 80, i32 144, i32 208, i32 17, i32 81, i32 145, i32 209, i32 18, i32 82, i32 146, i32 210, i32 19, i32 83, i32 147, i32 211, i32 20, i32 84, i32 148, i32 212, i32 21, i32 85, i32 149, i32 213, i32 22, i32 86, i32 150, i32 214, i32 23, i32 87, i32 151, i32 215, i32 24, i32 88, i32 152, i32 216, i32 25, i32 89, i32 153, i32 217, i32 26, i32 90, i32 154, i32 218, i32 27, i32 91, i32 155, i32 219, i32 28, i32 92, i32 156, i32 220, i32 29, i32 93, i32 157, i32 221, i32 30, i32 94, i32 158, i32 222, i32 31, i32 95, i32 159, i32 223, i32 32, i32 96, i32 160, i32 224, i32 33, i32 97, i32 161, i32 225, i32 34, i32 98, i32 162, i32 226, i32 35, i32 99, i32 163, i32 227, i32 36, i32 100, i32 164, i32 228, i32 37, i32 101, i32 165, i32 229, i32 38, i32 102, i32 166, i32 230, i32 39, i32 103, i32 167, i32 231, i32 40, i32 104, i32 168, i32 232, i32 41, i32 105, i32 169, i32 233, i32 42, i32 106, i32 170, i32 234, i32 43, i32 107, i32 171, i32 235, i32 44, i32 108, i32 172, i32 236, i32 45, i32 109, i32 173, i32 237, i32 46, i32 110, i32 174, i32 238, i32 47, i32 111, i32 175, i32 239, i32 48, i32 112, i32 176, i32 240, i32 49, i32 113, i32 177, i32 241, i32 50, i32 114, i32 178, i32 242, i32 51, i32 115, i32 179, i32 243, i32 52, i32 116, i32 180, i32 244, i32 53, i32 117, i32 181, i32 245, i32 54, i32 118, i32 182, i32 246, i32 55, i32 119, i32 183, i32 247, i32 56, i32 120, i32 184, i32 248, i32 57, i32 121, i32 185, i32 249, i32 58, i32 122, i32 186, i32 250, i32 59, i32 123, i32 187, i32 251, i32 60, i32 124, i32 188, i32 252, i32 61, i32 125, i32 189, i32 253, i32 62, i32 126, i32 190, i32 254, i32 63, i32 127, i32 191, i32 255>3936  store <256 x i16> %interleaved.vec, ptr %out.vec, align 643937  ret void3938}3939