brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE3; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx  | FileCheck %s --check-prefixes=AVX4; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX25; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FP6; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2-FCP7; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX5128; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512-FCP9; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ10; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-FCP11; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW12; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW-FCP13; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512DQ-BW14; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ-BW-FCP15 16; These patterns are produced by LoopVectorizer for interleaved stores.17 18define void @store_i16_stride8_vf2(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %in.vecptr4, ptr %in.vecptr5, ptr %in.vecptr6, ptr %in.vecptr7, ptr %out.vec) nounwind {19; SSE-LABEL: store_i16_stride8_vf2:20; SSE:       # %bb.0:21; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %rax22; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %r1023; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %r1124; SSE-NEXT:    movdqa (%rdi), %xmm025; SSE-NEXT:    movdqa (%rdx), %xmm126; SSE-NEXT:    movdqa (%r8), %xmm227; SSE-NEXT:    movdqa (%r11), %xmm328; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]29; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]30; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]31; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]32; SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[1],mem[1]33; SSE-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]34; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]35; SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,1,3,4,5,6,7]36; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,1,2,0]37; SSE-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,4,7,5]38; SSE-NEXT:    movsd {{.*#+}} xmm3 = xmm1[0],xmm3[1]39; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]40; SSE-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[2,0,3,1,4,5,6,7]41; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,1,1,3]42; SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,5,7]43; SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]44; SSE-NEXT:    movapd %xmm1, 16(%rax)45; SSE-NEXT:    movapd %xmm3, (%rax)46; SSE-NEXT:    retq47;48; AVX-LABEL: store_i16_stride8_vf2:49; AVX:       # %bb.0:50; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax51; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r1052; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r1153; AVX-NEXT:    vmovdqa (%rdi), %xmm054; AVX-NEXT:    vmovdqa (%rdx), %xmm155; AVX-NEXT:    vmovdqa (%r8), %xmm256; AVX-NEXT:    vmovdqa (%r11), %xmm357; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]58; AVX-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]59; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]60; AVX-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],mem[0],xmm2[1],mem[1]61; AVX-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],mem[0],xmm3[1],mem[1]62; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]63; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm264; AVX-NEXT:    vpblendw {{.*#+}} xmm3 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]65; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]66; AVX-NEXT:    vpackusdw %xmm3, %xmm2, %xmm267; AVX-NEXT:    vpsrld $16, %xmm1, %xmm168; AVX-NEXT:    vpsrld $16, %xmm0, %xmm069; AVX-NEXT:    vpackusdw %xmm1, %xmm0, %xmm070; AVX-NEXT:    vmovdqa %xmm0, 16(%rax)71; AVX-NEXT:    vmovdqa %xmm2, (%rax)72; AVX-NEXT:    retq73;74; AVX2-LABEL: store_i16_stride8_vf2:75; AVX2:       # %bb.0:76; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %rax77; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r1078; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r1179; AVX2-NEXT:    vmovdqa (%rdi), %xmm080; AVX2-NEXT:    vmovdqa (%rdx), %xmm181; AVX2-NEXT:    vmovdqa (%r8), %xmm282; AVX2-NEXT:    vmovdqa (%r11), %xmm383; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]84; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]85; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]86; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[1],mem[1]87; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm188; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm089; AVX2-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]90; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,2,3,6,7,10,11,14,15,16,17,20,21,24,25,28,29,18,19,22,23,26,27,30,31]91; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]92; AVX2-NEXT:    vmovdqa %ymm0, (%rax)93; AVX2-NEXT:    vzeroupper94; AVX2-NEXT:    retq95;96; AVX2-FP-LABEL: store_i16_stride8_vf2:97; AVX2-FP:       # %bb.0:98; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %rax99; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %r10100; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %r11101; AVX2-FP-NEXT:    vmovdqa (%rdi), %xmm0102; AVX2-FP-NEXT:    vmovdqa (%rdx), %xmm1103; AVX2-FP-NEXT:    vmovdqa (%r8), %xmm2104; AVX2-FP-NEXT:    vmovdqa (%r11), %xmm3105; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]106; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]107; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]108; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[1],mem[1]109; AVX2-FP-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1110; AVX2-FP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0111; AVX2-FP-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]112; AVX2-FP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,2,3,6,7,10,11,14,15,16,17,20,21,24,25,28,29,18,19,22,23,26,27,30,31]113; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]114; AVX2-FP-NEXT:    vmovdqa %ymm0, (%rax)115; AVX2-FP-NEXT:    vzeroupper116; AVX2-FP-NEXT:    retq117;118; AVX2-FCP-LABEL: store_i16_stride8_vf2:119; AVX2-FCP:       # %bb.0:120; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax121; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10122; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r11123; AVX2-FCP-NEXT:    vmovdqa (%rdi), %xmm0124; AVX2-FCP-NEXT:    vmovdqa (%rdx), %xmm1125; AVX2-FCP-NEXT:    vmovdqa (%r8), %xmm2126; AVX2-FCP-NEXT:    vmovdqa (%r11), %xmm3127; AVX2-FCP-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]128; AVX2-FCP-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]129; AVX2-FCP-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]130; AVX2-FCP-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[1],mem[1]131; AVX2-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1132; AVX2-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0133; AVX2-FCP-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]134; AVX2-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,2,3,6,7,10,11,14,15,16,17,20,21,24,25,28,29,18,19,22,23,26,27,30,31]135; AVX2-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]136; AVX2-FCP-NEXT:    vmovdqa %ymm0, (%rax)137; AVX2-FCP-NEXT:    vzeroupper138; AVX2-FCP-NEXT:    retq139;140; AVX512-LABEL: store_i16_stride8_vf2:141; AVX512:       # %bb.0:142; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax143; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10144; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r11145; AVX512-NEXT:    vmovdqa (%rdi), %xmm0146; AVX512-NEXT:    vmovdqa (%rdx), %xmm1147; AVX512-NEXT:    vmovdqa (%r8), %xmm2148; AVX512-NEXT:    vmovdqa (%r11), %xmm3149; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]150; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]151; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]152; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[1],mem[1]153; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1154; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0155; AVX512-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]156; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,2,3,6,7,10,11,14,15,16,17,20,21,24,25,28,29,18,19,22,23,26,27,30,31]157; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]158; AVX512-NEXT:    vmovdqa %ymm0, (%rax)159; AVX512-NEXT:    vzeroupper160; AVX512-NEXT:    retq161;162; AVX512-FCP-LABEL: store_i16_stride8_vf2:163; AVX512-FCP:       # %bb.0:164; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax165; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10166; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r11167; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm0168; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm1169; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm2170; AVX512-FCP-NEXT:    vmovdqa (%r11), %xmm3171; AVX512-FCP-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]172; AVX512-FCP-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]173; AVX512-FCP-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]174; AVX512-FCP-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[1],mem[1]175; AVX512-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1176; AVX512-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0177; AVX512-FCP-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]178; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,2,3,6,7,10,11,14,15,16,17,20,21,24,25,28,29,18,19,22,23,26,27,30,31]179; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]180; AVX512-FCP-NEXT:    vmovdqa %ymm0, (%rax)181; AVX512-FCP-NEXT:    vzeroupper182; AVX512-FCP-NEXT:    retq183;184; AVX512DQ-LABEL: store_i16_stride8_vf2:185; AVX512DQ:       # %bb.0:186; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax187; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %r10188; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %r11189; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0190; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm1191; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm2192; AVX512DQ-NEXT:    vmovdqa (%r11), %xmm3193; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]194; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]195; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]196; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[1],mem[1]197; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1198; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0199; AVX512DQ-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]200; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,2,3,6,7,10,11,14,15,16,17,20,21,24,25,28,29,18,19,22,23,26,27,30,31]201; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]202; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rax)203; AVX512DQ-NEXT:    vzeroupper204; AVX512DQ-NEXT:    retq205;206; AVX512DQ-FCP-LABEL: store_i16_stride8_vf2:207; AVX512DQ-FCP:       # %bb.0:208; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax209; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10210; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r11211; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm0212; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm1213; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm2214; AVX512DQ-FCP-NEXT:    vmovdqa (%r11), %xmm3215; AVX512DQ-FCP-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]216; AVX512DQ-FCP-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]217; AVX512DQ-FCP-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]218; AVX512DQ-FCP-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[1],mem[1]219; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1220; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0221; AVX512DQ-FCP-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]222; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,2,3,6,7,10,11,14,15,16,17,20,21,24,25,28,29,18,19,22,23,26,27,30,31]223; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]224; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, (%rax)225; AVX512DQ-FCP-NEXT:    vzeroupper226; AVX512DQ-FCP-NEXT:    retq227;228; AVX512BW-LABEL: store_i16_stride8_vf2:229; AVX512BW:       # %bb.0:230; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax231; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10232; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r11233; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0234; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm1235; AVX512BW-NEXT:    vmovdqa (%r8), %xmm2236; AVX512BW-NEXT:    vmovdqa (%r11), %xmm3237; AVX512BW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]238; AVX512BW-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]239; AVX512BW-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]240; AVX512BW-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[1],mem[1]241; AVX512BW-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1242; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0243; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} ymm2 = [0,2,16,18,8,10,24,26,1,3,17,19,9,11,25,27]244; AVX512BW-NEXT:    vpermi2w %ymm1, %ymm0, %ymm2245; AVX512BW-NEXT:    vmovdqa %ymm2, (%rax)246; AVX512BW-NEXT:    vzeroupper247; AVX512BW-NEXT:    retq248;249; AVX512BW-FCP-LABEL: store_i16_stride8_vf2:250; AVX512BW-FCP:       # %bb.0:251; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax252; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10253; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r11254; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %xmm0255; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %xmm1256; AVX512BW-FCP-NEXT:    vmovdqa (%r8), %xmm2257; AVX512BW-FCP-NEXT:    vmovdqa (%r11), %xmm3258; AVX512BW-FCP-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]259; AVX512BW-FCP-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]260; AVX512BW-FCP-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]261; AVX512BW-FCP-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[1],mem[1]262; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1263; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0264; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm2 = [0,2,16,18,8,10,24,26,1,3,17,19,9,11,25,27]265; AVX512BW-FCP-NEXT:    vpermi2w %ymm1, %ymm0, %ymm2266; AVX512BW-FCP-NEXT:    vmovdqa %ymm2, (%rax)267; AVX512BW-FCP-NEXT:    vzeroupper268; AVX512BW-FCP-NEXT:    retq269;270; AVX512DQ-BW-LABEL: store_i16_stride8_vf2:271; AVX512DQ-BW:       # %bb.0:272; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax273; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10274; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %r11275; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %xmm0276; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %xmm1277; AVX512DQ-BW-NEXT:    vmovdqa (%r8), %xmm2278; AVX512DQ-BW-NEXT:    vmovdqa (%r11), %xmm3279; AVX512DQ-BW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]280; AVX512DQ-BW-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]281; AVX512DQ-BW-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]282; AVX512DQ-BW-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[1],mem[1]283; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1284; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0285; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} ymm2 = [0,2,16,18,8,10,24,26,1,3,17,19,9,11,25,27]286; AVX512DQ-BW-NEXT:    vpermi2w %ymm1, %ymm0, %ymm2287; AVX512DQ-BW-NEXT:    vmovdqa %ymm2, (%rax)288; AVX512DQ-BW-NEXT:    vzeroupper289; AVX512DQ-BW-NEXT:    retq290;291; AVX512DQ-BW-FCP-LABEL: store_i16_stride8_vf2:292; AVX512DQ-BW-FCP:       # %bb.0:293; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax294; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10295; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r11296; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %xmm0297; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %xmm1298; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r8), %xmm2299; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r11), %xmm3300; AVX512DQ-BW-FCP-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]301; AVX512DQ-BW-FCP-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]302; AVX512DQ-BW-FCP-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[1],mem[1]303; AVX512DQ-BW-FCP-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[1],mem[1]304; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1305; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0306; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm2 = [0,2,16,18,8,10,24,26,1,3,17,19,9,11,25,27]307; AVX512DQ-BW-FCP-NEXT:    vpermi2w %ymm1, %ymm0, %ymm2308; AVX512DQ-BW-FCP-NEXT:    vmovdqa %ymm2, (%rax)309; AVX512DQ-BW-FCP-NEXT:    vzeroupper310; AVX512DQ-BW-FCP-NEXT:    retq311  %in.vec0 = load <2 x i16>, ptr %in.vecptr0, align 64312  %in.vec1 = load <2 x i16>, ptr %in.vecptr1, align 64313  %in.vec2 = load <2 x i16>, ptr %in.vecptr2, align 64314  %in.vec3 = load <2 x i16>, ptr %in.vecptr3, align 64315  %in.vec4 = load <2 x i16>, ptr %in.vecptr4, align 64316  %in.vec5 = load <2 x i16>, ptr %in.vecptr5, align 64317  %in.vec6 = load <2 x i16>, ptr %in.vecptr6, align 64318  %in.vec7 = load <2 x i16>, ptr %in.vecptr7, align 64319  %1 = shufflevector <2 x i16> %in.vec0, <2 x i16> %in.vec1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>320  %2 = shufflevector <2 x i16> %in.vec2, <2 x i16> %in.vec3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>321  %3 = shufflevector <2 x i16> %in.vec4, <2 x i16> %in.vec5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>322  %4 = shufflevector <2 x i16> %in.vec6, <2 x i16> %in.vec7, <4 x i32> <i32 0, i32 1, i32 2, i32 3>323  %5 = shufflevector <4 x i16> %1, <4 x i16> %2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>324  %6 = shufflevector <4 x i16> %3, <4 x i16> %4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>325  %7 = shufflevector <8 x i16> %5, <8 x i16> %6, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>326  %interleaved.vec = shufflevector <16 x i16> %7, <16 x i16> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>327  store <16 x i16> %interleaved.vec, ptr %out.vec, align 64328  ret void329}330 331define void @store_i16_stride8_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %in.vecptr4, ptr %in.vecptr5, ptr %in.vecptr6, ptr %in.vecptr7, ptr %out.vec) nounwind {332; SSE-LABEL: store_i16_stride8_vf4:333; SSE:       # %bb.0:334; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %rax335; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %r10336; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %r11337; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero338; SSE-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero339; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]340; SSE-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero341; SSE-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero342; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]343; SSE-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero344; SSE-NEXT:    movq {{.*#+}} xmm3 = mem[0],zero345; SSE-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]346; SSE-NEXT:    movq {{.*#+}} xmm3 = mem[0],zero347; SSE-NEXT:    movq {{.*#+}} xmm4 = mem[0],zero348; SSE-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]349; SSE-NEXT:    movdqa %xmm0, %xmm4350; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]351; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,0,0]352; SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,0,0]353; SSE-NEXT:    punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm5[2],xmm6[3],xmm5[3]354; SSE-NEXT:    movsd {{.*#+}} xmm6 = xmm4[0],xmm6[1]355; SSE-NEXT:    movdqa %xmm2, %xmm4356; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]357; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]358; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,1,1]359; SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]360; SSE-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,1],xmm4[2,3]361; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[3,3,3,3]362; SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]363; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[2,2,2,2]364; SSE-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[2,2,2,2]365; SSE-NEXT:    punpckhdq {{.*#+}} xmm8 = xmm8[2],xmm5[2],xmm8[3],xmm5[3]366; SSE-NEXT:    movsd {{.*#+}} xmm8 = xmm0[0],xmm8[1]367; SSE-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]368; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]369; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]370; SSE-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,1],xmm2[2,3]371; SSE-NEXT:    movaps %xmm4, 48(%rax)372; SSE-NEXT:    movapd %xmm8, 32(%rax)373; SSE-NEXT:    movaps %xmm7, 16(%rax)374; SSE-NEXT:    movapd %xmm6, (%rax)375; SSE-NEXT:    retq376;377; AVX-LABEL: store_i16_stride8_vf4:378; AVX:       # %bb.0:379; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax380; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r10381; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero382; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill383; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero384; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill385; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm1[0],xmm0[0]386; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero387; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill388; AVX-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero389; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm7 = xmm3[0],xmm0[0]390; AVX-NEXT:    vmovq {{.*#+}} xmm5 = mem[0],zero391; AVX-NEXT:    vmovq {{.*#+}} xmm6 = mem[0],zero392; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm8 = xmm6[0],xmm5[0]393; AVX-NEXT:    vmovq {{.*#+}} xmm9 = mem[0],zero394; AVX-NEXT:    vmovq {{.*#+}} xmm10 = mem[0],zero395; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm11 = xmm10[0],xmm9[0]396; AVX-NEXT:    vpshufd {{.*#+}} xmm12 = xmm11[3,1,2,3]397; AVX-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm12[0,1,3,1,4,5,6,7]398; AVX-NEXT:    vpshufd {{.*#+}} xmm14 = xmm8[3,1,2,3]399; AVX-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm14[0,1,3,1,4,5,6,7]400; AVX-NEXT:    vpunpckldq {{.*#+}} xmm13 = xmm15[0],xmm13[0],xmm15[1],xmm13[1]401; AVX-NEXT:    vpshufd {{.*#+}} xmm15 = xmm7[3,1,2,3]402; AVX-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm15[3,1,2,3,4,5,6,7]403; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm4[3,1,2,3]404; AVX-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm0[3,1,2,3,4,5,6,7]405; AVX-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]406; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm13[4,5,6,7]407; AVX-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm12[0,1,2,0,4,5,6,7]408; AVX-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm14[0,1,2,0,4,5,6,7]409; AVX-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm12[0],xmm2[0],xmm12[1],xmm2[1]410; AVX-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm15[2,0,2,3,4,5,6,7]411; AVX-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[2,0,2,3,4,5,6,7]412; AVX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm12[0],xmm0[1],xmm12[1]413; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]414; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0415; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1416; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm11[0],xmm1[1,2,3],xmm11[4],xmm1[5,6,7]417; AVX-NEXT:    vpblendw {{.*#+}} xmm8 = xmm8[0],xmm1[1,2,3],xmm8[4],xmm1[5,6,7]418; AVX-NEXT:    vpackusdw %xmm2, %xmm8, %xmm2419; AVX-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0],xmm1[1,2,3],xmm7[4],xmm1[5,6,7]420; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm4[0],xmm1[1,2,3],xmm4[4],xmm1[5,6,7]421; AVX-NEXT:    vpackusdw %xmm7, %xmm1, %xmm1422; AVX-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1423; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]424; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]425; AVX-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]426; AVX-NEXT:    vpunpckldq {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload427; AVX-NEXT:    # xmm3 = xmm3[0],mem[0],xmm3[1],mem[1]428; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload429; AVX-NEXT:    vpunpckldq {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload430; AVX-NEXT:    # xmm4 = xmm4[0],mem[0],xmm4[1],mem[1]431; AVX-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[1,3,2,3,4,5,6,7]432; AVX-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]433; AVX-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]434; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]435; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax436; AVX-NEXT:    vmovdqa %xmm2, 16(%rax)437; AVX-NEXT:    vmovdqa %xmm1, (%rax)438; AVX-NEXT:    vmovaps %ymm0, 32(%rax)439; AVX-NEXT:    vzeroupper440; AVX-NEXT:    retq441;442; AVX2-LABEL: store_i16_stride8_vf4:443; AVX2:       # %bb.0:444; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %rax445; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r10446; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r11447; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero448; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero449; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]450; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero451; AVX2-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero452; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]453; AVX2-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero454; AVX2-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero455; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]456; AVX2-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero457; AVX2-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero458; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]459; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm4460; AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm5461; AVX2-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2462; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm2[0,1,2,0,4,5,6,4]463; AVX2-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,7,5,6,4,8,9,10,11,15,13,14,12]464; AVX2-NEXT:    vpshufd {{.*#+}} ymm6 = ymm5[0,1,2,0,4,5,6,4]465; AVX2-NEXT:    vpshufhw {{.*#+}} ymm6 = ymm6[0,1,2,3,6,4,7,5,8,9,10,11,14,12,15,13]466; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0,1,2],ymm3[3],ymm6[4,5],ymm3[6],ymm6[7]467; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0468; AVX2-NEXT:    vpshufd {{.*#+}} ymm1 = ymm0[0,2,2,3,4,6,6,7]469; AVX2-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[1,3,0,2,4,5,6,7,9,11,8,10,12,13,14,15]470; AVX2-NEXT:    vpshufd {{.*#+}} ymm6 = ymm4[0,2,2,3,4,6,6,7]471; AVX2-NEXT:    vpshuflw {{.*#+}} ymm6 = ymm6[0,2,1,3,4,5,6,7,8,10,9,11,12,13,14,15]472; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm6[0],ymm1[1],ymm6[2,3],ymm1[4],ymm6[5,6,7]473; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm3[2,3],ymm1[4,5],ymm3[6,7]474; AVX2-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,1,1,3,4,5,5,7]475; AVX2-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,5,7,4,6,8,9,10,11,13,15,12,14]476; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm5[0,1,1,3,4,5,5,7]477; AVX2-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,6,5,7,8,9,10,11,12,14,13,15]478; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5],ymm2[6],ymm3[7]479; AVX2-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[3,1,2,3,7,5,6,7]480; AVX2-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[3,1,2,0,4,5,6,7,11,9,10,8,12,13,14,15]481; AVX2-NEXT:    vpshufd {{.*#+}} ymm3 = ymm4[3,1,2,3,7,5,6,7]482; AVX2-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm3[2,0,3,1,4,5,6,7,10,8,11,9,12,13,14,15]483; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0],ymm0[1],ymm3[2,3],ymm0[4],ymm3[5,6,7]484; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2,3],ymm0[4,5],ymm2[6,7]485; AVX2-NEXT:    vmovdqa %ymm0, 32(%rax)486; AVX2-NEXT:    vmovdqa %ymm1, (%rax)487; AVX2-NEXT:    vzeroupper488; AVX2-NEXT:    retq489;490; AVX2-FP-LABEL: store_i16_stride8_vf4:491; AVX2-FP:       # %bb.0:492; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %rax493; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %r10494; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %r11495; AVX2-FP-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero496; AVX2-FP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero497; AVX2-FP-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]498; AVX2-FP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero499; AVX2-FP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero500; AVX2-FP-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]501; AVX2-FP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero502; AVX2-FP-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero503; AVX2-FP-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]504; AVX2-FP-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero505; AVX2-FP-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero506; AVX2-FP-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]507; AVX2-FP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm4508; AVX2-FP-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm5509; AVX2-FP-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm2510; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,2,3,4,5,6,7,0,1,8,9,0,1,8,9,16,17,18,19,20,21,22,23,18,19,26,27,18,19,26,27]511; AVX2-FP-NEXT:    vpshufb %ymm3, %ymm2, %ymm6512; AVX2-FP-NEXT:    vpshufb %ymm3, %ymm5, %ymm3513; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm6[3],ymm3[4,5],ymm6[6],ymm3[7]514; AVX2-FP-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0515; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,8,9,0,1,8,9,8,9,10,11,12,13,14,15,18,19,26,27,18,19,26,27,24,25,26,27,28,29,30,31]516; AVX2-FP-NEXT:    vpshufb %ymm1, %ymm0, %ymm6517; AVX2-FP-NEXT:    vpshufb %ymm1, %ymm4, %ymm1518; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],ymm6[1],ymm1[2,3],ymm6[4],ymm1[5,6,7]519; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm3[2,3],ymm1[4,5],ymm3[6,7]520; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,2,3,4,5,6,7,4,5,12,13,4,5,12,13,16,17,18,19,20,21,22,23,22,23,30,31,22,23,30,31]521; AVX2-FP-NEXT:    vpshufb %ymm3, %ymm2, %ymm2522; AVX2-FP-NEXT:    vpshufb %ymm3, %ymm5, %ymm3523; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5],ymm2[6],ymm3[7]524; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm3 = [4,5,12,13,4,5,12,13,8,9,10,11,12,13,14,15,22,23,30,31,22,23,30,31,24,25,26,27,28,29,30,31]525; AVX2-FP-NEXT:    vpshufb %ymm3, %ymm0, %ymm0526; AVX2-FP-NEXT:    vpshufb %ymm3, %ymm4, %ymm3527; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0],ymm0[1],ymm3[2,3],ymm0[4],ymm3[5,6,7]528; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2,3],ymm0[4,5],ymm2[6,7]529; AVX2-FP-NEXT:    vmovdqa %ymm0, 32(%rax)530; AVX2-FP-NEXT:    vmovdqa %ymm1, (%rax)531; AVX2-FP-NEXT:    vzeroupper532; AVX2-FP-NEXT:    retq533;534; AVX2-FCP-LABEL: store_i16_stride8_vf4:535; AVX2-FCP:       # %bb.0:536; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax537; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10538; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r11539; AVX2-FCP-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero540; AVX2-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero541; AVX2-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]542; AVX2-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero543; AVX2-FCP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero544; AVX2-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]545; AVX2-FCP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero546; AVX2-FCP-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero547; AVX2-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]548; AVX2-FCP-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero549; AVX2-FCP-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero550; AVX2-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]551; AVX2-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0552; AVX2-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm1553; AVX2-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [0,2,4,6,0,2,4,6]554; AVX2-FCP-NEXT:    # ymm2 = mem[0,1,0,1]555; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm2, %ymm3556; AVX2-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [u,u,u,u,u,u,u,u,0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,2,3,6,7,10,11,14,15]557; AVX2-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3558; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm2, %ymm2559; AVX2-FCP-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]560; AVX2-FCP-NEXT:    vpshufb %ymm5, %ymm2, %ymm2561; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5],ymm3[6,7]562; AVX2-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [1,3,5,7,1,3,5,7]563; AVX2-FCP-NEXT:    # ymm3 = mem[0,1,0,1]564; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm1565; AVX2-FCP-NEXT:    vpshufb %ymm4, %ymm1, %ymm1566; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm3, %ymm0567; AVX2-FCP-NEXT:    vpshufb %ymm5, %ymm0, %ymm0568; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]569; AVX2-FCP-NEXT:    vmovdqa %ymm0, 32(%rax)570; AVX2-FCP-NEXT:    vmovdqa %ymm2, (%rax)571; AVX2-FCP-NEXT:    vzeroupper572; AVX2-FCP-NEXT:    retq573;574; AVX512-LABEL: store_i16_stride8_vf4:575; AVX512:       # %bb.0:576; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax577; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10578; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r11579; AVX512-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero580; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero581; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]582; AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero583; AVX512-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero584; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]585; AVX512-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero586; AVX512-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero587; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]588; AVX512-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero589; AVX512-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero590; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]591; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm4592; AVX512-NEXT:    vpshufd {{.*#+}} ymm5 = ymm4[0,1,1,3,4,5,5,7]593; AVX512-NEXT:    vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,5,7,4,6,8,9,10,11,13,15,12,14]594; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2595; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm2[0,1,1,3,4,5,5,7]596; AVX512-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,6,5,7,8,9,10,11,12,14,13,15]597; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3],ymm3[4,5],ymm5[6],ymm3[7]598; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm5599; AVX512-NEXT:    vpshufd {{.*#+}} ymm6 = ymm5[3,1,2,3,7,5,6,7]600; AVX512-NEXT:    vpshuflw {{.*#+}} ymm6 = ymm6[3,1,2,0,4,5,6,7,11,9,10,8,12,13,14,15]601; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0602; AVX512-NEXT:    vpshufd {{.*#+}} ymm1 = ymm0[3,1,2,3,7,5,6,7]603; AVX512-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[2,0,3,1,4,5,6,7,10,8,11,9,12,13,14,15]604; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],ymm6[1],ymm1[2,3],ymm6[4],ymm1[5,6,7]605; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm3[2,3],ymm1[4,5],ymm3[6,7]606; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm4[0,1,2,0,4,5,6,4]607; AVX512-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,7,5,6,4,8,9,10,11,15,13,14,12]608; AVX512-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4]609; AVX512-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,6,4,7,5,8,9,10,11,14,12,15,13]610; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7]611; AVX512-NEXT:    vpshufd {{.*#+}} ymm3 = ymm5[0,2,2,3,4,6,6,7]612; AVX512-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm3[1,3,0,2,4,5,6,7,9,11,8,10,12,13,14,15]613; AVX512-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,2,2,3,4,6,6,7]614; AVX512-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,2,1,3,4,5,6,7,8,10,9,11,12,13,14,15]615; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2,3],ymm3[4],ymm0[5,6,7]616; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2,3],ymm0[4,5],ymm2[6,7]617; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0618; AVX512-NEXT:    vmovdqa64 %zmm0, (%rax)619; AVX512-NEXT:    vzeroupper620; AVX512-NEXT:    retq621;622; AVX512-FCP-LABEL: store_i16_stride8_vf4:623; AVX512-FCP:       # %bb.0:624; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax625; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10626; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r11627; AVX512-FCP-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero628; AVX512-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero629; AVX512-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]630; AVX512-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero631; AVX512-FCP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero632; AVX512-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]633; AVX512-FCP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero634; AVX512-FCP-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero635; AVX512-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]636; AVX512-FCP-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero637; AVX512-FCP-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero638; AVX512-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]639; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [1,3,9,11,1,3,9,11]640; AVX512-FCP-NEXT:    # ymm4 = mem[0,1,0,1]641; AVX512-FCP-NEXT:    vmovdqa %ymm2, %ymm5642; AVX512-FCP-NEXT:    vpermt2d %ymm3, %ymm4, %ymm5643; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [u,u,u,u,u,u,u,u,0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,2,3,6,7,10,11,14,15]644; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm5, %ymm5645; AVX512-FCP-NEXT:    vpermi2d %ymm1, %ymm0, %ymm4646; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]647; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm4, %ymm4648; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm5[2,3],ymm4[4,5],ymm5[6,7]649; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [0,2,8,10,0,2,8,10]650; AVX512-FCP-NEXT:    # ymm5 = mem[0,1,0,1]651; AVX512-FCP-NEXT:    vpermt2d %ymm3, %ymm5, %ymm2652; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm2, %ymm2653; AVX512-FCP-NEXT:    vpermt2d %ymm1, %ymm5, %ymm0654; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm0, %ymm0655; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2,3],ymm0[4,5],ymm2[6,7]656; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0657; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, (%rax)658; AVX512-FCP-NEXT:    vzeroupper659; AVX512-FCP-NEXT:    retq660;661; AVX512DQ-LABEL: store_i16_stride8_vf4:662; AVX512DQ:       # %bb.0:663; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax664; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %r10665; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %r11666; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero667; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero668; AVX512DQ-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]669; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero670; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero671; AVX512DQ-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]672; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero673; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero674; AVX512DQ-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]675; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero676; AVX512DQ-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero677; AVX512DQ-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]678; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm3, %ymm4679; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm5 = ymm4[0,1,1,3,4,5,5,7]680; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,5,7,4,6,8,9,10,11,13,15,12,14]681; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2682; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm2[0,1,1,3,4,5,5,7]683; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,6,5,7,8,9,10,11,12,14,13,15]684; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3],ymm3[4,5],ymm5[6],ymm3[7]685; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm5686; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm6 = ymm5[3,1,2,3,7,5,6,7]687; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm6 = ymm6[3,1,2,0,4,5,6,7,11,9,10,8,12,13,14,15]688; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0689; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm1 = ymm0[3,1,2,3,7,5,6,7]690; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm1 = ymm1[2,0,3,1,4,5,6,7,10,8,11,9,12,13,14,15]691; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],ymm6[1],ymm1[2,3],ymm6[4],ymm1[5,6,7]692; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm3[2,3],ymm1[4,5],ymm3[6,7]693; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm4[0,1,2,0,4,5,6,4]694; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,7,5,6,4,8,9,10,11,15,13,14,12]695; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,1,2,0,4,5,6,4]696; AVX512DQ-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,6,4,7,5,8,9,10,11,14,12,15,13]697; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7]698; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm3 = ymm5[0,2,2,3,4,6,6,7]699; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm3 = ymm3[1,3,0,2,4,5,6,7,9,11,8,10,12,13,14,15]700; AVX512DQ-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[0,2,2,3,4,6,6,7]701; AVX512DQ-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[0,2,1,3,4,5,6,7,8,10,9,11,12,13,14,15]702; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2,3],ymm3[4],ymm0[5,6,7]703; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2,3],ymm0[4,5],ymm2[6,7]704; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0705; AVX512DQ-NEXT:    vmovdqa64 %zmm0, (%rax)706; AVX512DQ-NEXT:    vzeroupper707; AVX512DQ-NEXT:    retq708;709; AVX512DQ-FCP-LABEL: store_i16_stride8_vf4:710; AVX512DQ-FCP:       # %bb.0:711; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax712; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10713; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r11714; AVX512DQ-FCP-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero715; AVX512DQ-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero716; AVX512DQ-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]717; AVX512DQ-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero718; AVX512DQ-FCP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero719; AVX512DQ-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]720; AVX512DQ-FCP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero721; AVX512DQ-FCP-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero722; AVX512DQ-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]723; AVX512DQ-FCP-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero724; AVX512DQ-FCP-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero725; AVX512DQ-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]726; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [1,3,9,11,1,3,9,11]727; AVX512DQ-FCP-NEXT:    # ymm4 = mem[0,1,0,1]728; AVX512DQ-FCP-NEXT:    vmovdqa %ymm2, %ymm5729; AVX512DQ-FCP-NEXT:    vpermt2d %ymm3, %ymm4, %ymm5730; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [u,u,u,u,u,u,u,u,0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,2,3,6,7,10,11,14,15]731; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm5, %ymm5732; AVX512DQ-FCP-NEXT:    vpermi2d %ymm1, %ymm0, %ymm4733; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u]734; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm4, %ymm4735; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm5[2,3],ymm4[4,5],ymm5[6,7]736; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [0,2,8,10,0,2,8,10]737; AVX512DQ-FCP-NEXT:    # ymm5 = mem[0,1,0,1]738; AVX512DQ-FCP-NEXT:    vpermt2d %ymm3, %ymm5, %ymm2739; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm2, %ymm2740; AVX512DQ-FCP-NEXT:    vpermt2d %ymm1, %ymm5, %ymm0741; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm0, %ymm0742; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2,3],ymm0[4,5],ymm2[6,7]743; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm0744; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, (%rax)745; AVX512DQ-FCP-NEXT:    vzeroupper746; AVX512DQ-FCP-NEXT:    retq747;748; AVX512BW-LABEL: store_i16_stride8_vf4:749; AVX512BW:       # %bb.0:750; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax751; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10752; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r11753; AVX512BW-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero754; AVX512BW-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero755; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]756; AVX512BW-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero757; AVX512BW-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero758; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]759; AVX512BW-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero760; AVX512BW-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero761; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]762; AVX512BW-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero763; AVX512BW-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero764; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]765; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0766; AVX512BW-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm1767; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,4,8,12,32,36,40,44,1,5,9,13,33,37,41,45,2,6,10,14,34,38,42,46,3,7,11,15,35,39,43,47]768; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm2769; AVX512BW-NEXT:    vmovdqa64 %zmm2, (%rax)770; AVX512BW-NEXT:    vzeroupper771; AVX512BW-NEXT:    retq772;773; AVX512BW-FCP-LABEL: store_i16_stride8_vf4:774; AVX512BW-FCP:       # %bb.0:775; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax776; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10777; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r11778; AVX512BW-FCP-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero779; AVX512BW-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero780; AVX512BW-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]781; AVX512BW-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero782; AVX512BW-FCP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero783; AVX512BW-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]784; AVX512BW-FCP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero785; AVX512BW-FCP-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero786; AVX512BW-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]787; AVX512BW-FCP-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero788; AVX512BW-FCP-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero789; AVX512BW-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]790; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0791; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm1792; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,4,8,12,32,36,40,44,1,5,9,13,33,37,41,45,2,6,10,14,34,38,42,46,3,7,11,15,35,39,43,47]793; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm2794; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm2, (%rax)795; AVX512BW-FCP-NEXT:    vzeroupper796; AVX512BW-FCP-NEXT:    retq797;798; AVX512DQ-BW-LABEL: store_i16_stride8_vf4:799; AVX512DQ-BW:       # %bb.0:800; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax801; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10802; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %r11803; AVX512DQ-BW-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero804; AVX512DQ-BW-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero805; AVX512DQ-BW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]806; AVX512DQ-BW-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero807; AVX512DQ-BW-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero808; AVX512DQ-BW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]809; AVX512DQ-BW-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero810; AVX512DQ-BW-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero811; AVX512DQ-BW-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]812; AVX512DQ-BW-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero813; AVX512DQ-BW-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero814; AVX512DQ-BW-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]815; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0816; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm1817; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,4,8,12,32,36,40,44,1,5,9,13,33,37,41,45,2,6,10,14,34,38,42,46,3,7,11,15,35,39,43,47]818; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm2819; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm2, (%rax)820; AVX512DQ-BW-NEXT:    vzeroupper821; AVX512DQ-BW-NEXT:    retq822;823; AVX512DQ-BW-FCP-LABEL: store_i16_stride8_vf4:824; AVX512DQ-BW-FCP:       # %bb.0:825; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax826; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10827; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r11828; AVX512DQ-BW-FCP-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero829; AVX512DQ-BW-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero830; AVX512DQ-BW-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]831; AVX512DQ-BW-FCP-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero832; AVX512DQ-BW-FCP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero833; AVX512DQ-BW-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]834; AVX512DQ-BW-FCP-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero835; AVX512DQ-BW-FCP-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero836; AVX512DQ-BW-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]837; AVX512DQ-BW-FCP-NEXT:    vmovq {{.*#+}} xmm3 = mem[0],zero838; AVX512DQ-BW-FCP-NEXT:    vmovq {{.*#+}} xmm4 = mem[0],zero839; AVX512DQ-BW-FCP-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]840; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0841; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm1842; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,4,8,12,32,36,40,44,1,5,9,13,33,37,41,45,2,6,10,14,34,38,42,46,3,7,11,15,35,39,43,47]843; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm2844; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm2, (%rax)845; AVX512DQ-BW-FCP-NEXT:    vzeroupper846; AVX512DQ-BW-FCP-NEXT:    retq847  %in.vec0 = load <4 x i16>, ptr %in.vecptr0, align 64848  %in.vec1 = load <4 x i16>, ptr %in.vecptr1, align 64849  %in.vec2 = load <4 x i16>, ptr %in.vecptr2, align 64850  %in.vec3 = load <4 x i16>, ptr %in.vecptr3, align 64851  %in.vec4 = load <4 x i16>, ptr %in.vecptr4, align 64852  %in.vec5 = load <4 x i16>, ptr %in.vecptr5, align 64853  %in.vec6 = load <4 x i16>, ptr %in.vecptr6, align 64854  %in.vec7 = load <4 x i16>, ptr %in.vecptr7, align 64855  %1 = shufflevector <4 x i16> %in.vec0, <4 x i16> %in.vec1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>856  %2 = shufflevector <4 x i16> %in.vec2, <4 x i16> %in.vec3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>857  %3 = shufflevector <4 x i16> %in.vec4, <4 x i16> %in.vec5, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>858  %4 = shufflevector <4 x i16> %in.vec6, <4 x i16> %in.vec7, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>859  %5 = shufflevector <8 x i16> %1, <8 x i16> %2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>860  %6 = shufflevector <8 x i16> %3, <8 x i16> %4, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>861  %7 = shufflevector <16 x i16> %5, <16 x i16> %6, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>862  %interleaved.vec = shufflevector <32 x i16> %7, <32 x i16> poison, <32 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>863  store <32 x i16> %interleaved.vec, ptr %out.vec, align 64864  ret void865}866 867define void @store_i16_stride8_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %in.vecptr4, ptr %in.vecptr5, ptr %in.vecptr6, ptr %in.vecptr7, ptr %out.vec) nounwind {868; SSE-LABEL: store_i16_stride8_vf8:869; SSE:       # %bb.0:870; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %rax871; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %r10872; SSE-NEXT:    movdqa (%rdi), %xmm0873; SSE-NEXT:    movdqa (%rsi), %xmm9874; SSE-NEXT:    movdqa (%rdx), %xmm1875; SSE-NEXT:    movdqa (%rcx), %xmm11876; SSE-NEXT:    movdqa (%r8), %xmm4877; SSE-NEXT:    movdqa (%r9), %xmm8878; SSE-NEXT:    movdqa (%r10), %xmm3879; SSE-NEXT:    movdqa (%rax), %xmm10880; SSE-NEXT:    movdqa %xmm1, %xmm13881; SSE-NEXT:    punpcklwd {{.*#+}} xmm13 = xmm13[0],xmm11[0],xmm13[1],xmm11[1],xmm13[2],xmm11[2],xmm13[3],xmm11[3]882; SSE-NEXT:    movdqa %xmm0, %xmm12883; SSE-NEXT:    punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3]884; SSE-NEXT:    movdqa %xmm12, %xmm5885; SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm13[0],xmm5[1],xmm13[1]886; SSE-NEXT:    movdqa %xmm3, %xmm14887; SSE-NEXT:    punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]888; SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm14[0,0,0,0]889; SSE-NEXT:    movdqa %xmm4, %xmm15890; SSE-NEXT:    punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3]891; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm15[0,0,0,0]892; SSE-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm6[2],xmm2[3],xmm6[3]893; SSE-NEXT:    movsd {{.*#+}} xmm2 = xmm5[0],xmm2[1]894; SSE-NEXT:    movdqa %xmm15, %xmm6895; SSE-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm14[0],xmm6[1],xmm14[1]896; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm13[1,1,1,1]897; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[1,1,1,1]898; SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]899; SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm6[2,3]900; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm13[3,3,3,3]901; SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm12[3,3,3,3]902; SSE-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]903; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm15[2,2,2,2]904; SSE-NEXT:    punpckhdq {{.*#+}} xmm15 = xmm15[2],xmm14[2],xmm15[3],xmm14[3]905; SSE-NEXT:    shufps {{.*#+}} xmm6 = xmm6[0,1],xmm15[2,3]906; SSE-NEXT:    punpckhdq {{.*#+}} xmm12 = xmm12[2],xmm13[2],xmm12[3],xmm13[3]907; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm14[2,2,2,2]908; SSE-NEXT:    punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm13[2],xmm7[3],xmm13[3]909; SSE-NEXT:    movsd {{.*#+}} xmm7 = xmm12[0],xmm7[1]910; SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm11[4],xmm1[5],xmm11[5],xmm1[6],xmm11[6],xmm1[7],xmm11[7]911; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]912; SSE-NEXT:    movdqa %xmm0, %xmm9913; SSE-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm1[0],xmm9[1],xmm1[1]914; SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]915; SSE-NEXT:    pshufd {{.*#+}} xmm10 = xmm3[0,0,0,0]916; SSE-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm8[4],xmm4[5],xmm8[5],xmm4[6],xmm8[6],xmm4[7],xmm8[7]917; SSE-NEXT:    pshufd {{.*#+}} xmm8 = xmm4[0,0,0,0]918; SSE-NEXT:    punpckhdq {{.*#+}} xmm8 = xmm8[2],xmm10[2],xmm8[3],xmm10[3]919; SSE-NEXT:    movsd {{.*#+}} xmm8 = xmm9[0],xmm8[1]920; SSE-NEXT:    movdqa %xmm4, %xmm9921; SSE-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm3[0],xmm9[1],xmm3[1]922; SSE-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[1,1,1,1]923; SSE-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,1,1]924; SSE-NEXT:    punpckldq {{.*#+}} xmm11 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]925; SSE-NEXT:    shufps {{.*#+}} xmm11 = xmm11[0,1],xmm9[2,3]926; SSE-NEXT:    pshufd {{.*#+}} xmm9 = xmm1[3,3,3,3]927; SSE-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[3,3,3,3]928; SSE-NEXT:    punpckldq {{.*#+}} xmm10 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]929; SSE-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[2,2,2,2]930; SSE-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]931; SSE-NEXT:    shufps {{.*#+}} xmm10 = xmm10[0,1],xmm4[2,3]932; SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]933; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,2,2,2]934; SSE-NEXT:    punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm1[2],xmm9[3],xmm1[3]935; SSE-NEXT:    movsd {{.*#+}} xmm9 = xmm0[0],xmm9[1]936; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %rax937; SSE-NEXT:    movapd %xmm9, 96(%rax)938; SSE-NEXT:    movaps %xmm10, 112(%rax)939; SSE-NEXT:    movaps %xmm11, 80(%rax)940; SSE-NEXT:    movapd %xmm8, 64(%rax)941; SSE-NEXT:    movapd %xmm7, 32(%rax)942; SSE-NEXT:    movaps %xmm6, 48(%rax)943; SSE-NEXT:    movaps %xmm5, 16(%rax)944; SSE-NEXT:    movapd %xmm2, (%rax)945; SSE-NEXT:    retq946;947; AVX-LABEL: store_i16_stride8_vf8:948; AVX:       # %bb.0:949; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax950; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r10951; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r11952; AVX-NEXT:    vmovdqa (%rdi), %xmm3953; AVX-NEXT:    vmovdqa (%rsi), %xmm5954; AVX-NEXT:    vmovdqa (%rdx), %xmm7955; AVX-NEXT:    vmovdqa (%rcx), %xmm8956; AVX-NEXT:    vmovdqa (%r8), %xmm1957; AVX-NEXT:    vmovdqa (%r9), %xmm2958; AVX-NEXT:    vmovdqa (%r11), %xmm4959; AVX-NEXT:    vmovdqa (%r10), %xmm6960; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3]961; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]962; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm10[1,1,1,1]963; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm9[2,3],xmm0[4,5,6,7]964; AVX-NEXT:    vpunpckldq {{.*#+}} xmm11 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]965; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm11, %ymm0966; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]967; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]968; AVX-NEXT:    vpunpckldq {{.*#+}} xmm13 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]969; AVX-NEXT:    vpshufd {{.*#+}} xmm14 = xmm11[0,0,0,0]970; AVX-NEXT:    vpshufd {{.*#+}} xmm15 = xmm12[0,1,0,1]971; AVX-NEXT:    vpblendw {{.*#+}} xmm14 = xmm15[0,1,2,3,4,5],xmm14[6,7]972; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm14, %ymm13973; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm13[2,3],ymm0[4,5],ymm13[6,7]974; AVX-NEXT:    vpshufd {{.*#+}} xmm13 = xmm9[2,3,2,3]975; AVX-NEXT:    vpshufd {{.*#+}} xmm14 = xmm10[3,3,3,3]976; AVX-NEXT:    vpblendw {{.*#+}} xmm13 = xmm14[0,1],xmm13[2,3],xmm14[4,5,6,7]977; AVX-NEXT:    vpunpckhdq {{.*#+}} xmm9 = xmm10[2],xmm9[2],xmm10[3],xmm9[3]978; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm9, %ymm9979; AVX-NEXT:    vpunpckhdq {{.*#+}} xmm10 = xmm12[2],xmm11[2],xmm12[3],xmm11[3]980; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,2,2,2]981; AVX-NEXT:    vpblendw {{.*#+}} xmm11 = xmm12[0,1,2,3,4,5],xmm11[6,7]982; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm11, %ymm10983; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1],ymm10[2,3],ymm9[4,5],ymm10[6,7]984; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7]985; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]986; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]987; AVX-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm7[2,3],xmm5[4,5,6,7]988; AVX-NEXT:    vpunpckldq {{.*#+}} xmm8 = xmm3[0],xmm7[0],xmm3[1],xmm7[1]989; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm8, %ymm5990; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]991; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]992; AVX-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]993; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[0,0,0,0]994; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm1[0,1,0,1]995; AVX-NEXT:    vpblendw {{.*#+}} xmm6 = xmm8[0,1,2,3,4,5],xmm6[6,7]996; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm6, %ymm2997; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm5[0,1],ymm2[2,3],ymm5[4,5],ymm2[6,7]998; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm7[2,3,2,3]999; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[3,3,3,3]1000; AVX-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3],xmm6[4,5,6,7]1001; AVX-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm3[2],xmm7[2],xmm3[3],xmm7[3]1002; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm31003; AVX-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm1[2],xmm4[2],xmm1[3],xmm4[3]1004; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,2,2,2]1005; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm4[6,7]1006; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm11007; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,3],ymm3[4,5],ymm1[6,7]1008; AVX-NEXT:    vmovaps %ymm1, 96(%rax)1009; AVX-NEXT:    vmovaps %ymm2, 64(%rax)1010; AVX-NEXT:    vmovaps %ymm9, 32(%rax)1011; AVX-NEXT:    vmovaps %ymm0, (%rax)1012; AVX-NEXT:    vzeroupper1013; AVX-NEXT:    retq1014;1015; AVX2-LABEL: store_i16_stride8_vf8:1016; AVX2:       # %bb.0:1017; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %rax1018; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r101019; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r111020; AVX2-NEXT:    vmovdqa (%rdi), %xmm01021; AVX2-NEXT:    vmovdqa (%rdx), %xmm11022; AVX2-NEXT:    vmovdqa (%r8), %xmm21023; AVX2-NEXT:    vmovdqa (%r11), %xmm31024; AVX2-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm01025; AVX2-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm11026; AVX2-NEXT:    vinserti128 $1, (%r9), %ymm2, %ymm41027; AVX2-NEXT:    vinserti128 $1, (%r10), %ymm3, %ymm31028; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm3[0,2,0,2]1029; AVX2-NEXT:    vmovdqa {{.*#+}} ymm6 = [u,u,u,u,u,u,u,u,u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,u,u,u,u,2,3,10,11]1030; AVX2-NEXT:    vpshufb %ymm6, %ymm5, %ymm21031; AVX2-NEXT:    vpermq {{.*#+}} ymm7 = ymm4[0,2,0,2]1032; AVX2-NEXT:    vpmovsxdq {{.*#+}} ymm8 = [0,151519488,0,185205506]1033; AVX2-NEXT:    vpshufb %ymm8, %ymm7, %ymm91034; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm9[0,1,2],ymm2[3],ymm9[4,5,6],ymm2[7]1035; AVX2-NEXT:    vpermq {{.*#+}} ymm9 = ymm1[0,2,0,2]1036; AVX2-NEXT:    vmovdqa {{.*#+}} ymm10 = [u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,u,u,u,u,2,3,10,11,u,u,u,u,u,u,u,u]1037; AVX2-NEXT:    vpshufb %ymm10, %ymm9, %ymm111038; AVX2-NEXT:    vpermq {{.*#+}} ymm12 = ymm0[0,2,0,2]1039; AVX2-NEXT:    vpmovsxdq {{.*#+}} ymm13 = [151519488,0,185205506,0]1040; AVX2-NEXT:    vpshufb %ymm13, %ymm12, %ymm141041; AVX2-NEXT:    vpblendd {{.*#+}} ymm11 = ymm14[0],ymm11[1],ymm14[2,3,4],ymm11[5],ymm14[6,7]1042; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm11[0,1],ymm2[2,3],ymm11[4,5],ymm2[6,7]1043; AVX2-NEXT:    vmovdqa {{.*#+}} ymm11 = [u,u,u,u,u,u,u,u,u,u,u,u,4,5,12,13,u,u,u,u,u,u,u,u,u,u,u,u,6,7,14,15]1044; AVX2-NEXT:    vpshufb %ymm11, %ymm5, %ymm51045; AVX2-NEXT:    vpmovsxdq {{.*#+}} ymm14 = [0,218891524,0,252577542]1046; AVX2-NEXT:    vpshufb %ymm14, %ymm7, %ymm71047; AVX2-NEXT:    vpblendd {{.*#+}} ymm5 = ymm7[0,1,2],ymm5[3],ymm7[4,5,6],ymm5[7]1048; AVX2-NEXT:    vmovdqa {{.*#+}} ymm7 = [u,u,u,u,4,5,12,13,u,u,u,u,u,u,u,u,u,u,u,u,6,7,14,15,u,u,u,u,u,u,u,u]1049; AVX2-NEXT:    vpshufb %ymm7, %ymm9, %ymm91050; AVX2-NEXT:    vpmovsxdq {{.*#+}} ymm15 = [218891524,0,252577542,0]1051; AVX2-NEXT:    vpshufb %ymm15, %ymm12, %ymm121052; AVX2-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0],ymm9[1],ymm12[2,3,4],ymm9[5],ymm12[6,7]1053; AVX2-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1],ymm5[2,3],ymm9[4,5],ymm5[6,7]1054; AVX2-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[1,3,1,3]1055; AVX2-NEXT:    vpshufb %ymm6, %ymm3, %ymm61056; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[1,3,1,3]1057; AVX2-NEXT:    vpshufb %ymm8, %ymm4, %ymm81058; AVX2-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2],ymm6[3],ymm8[4,5,6],ymm6[7]1059; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]1060; AVX2-NEXT:    vpshufb %ymm10, %ymm1, %ymm81061; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,3,1,3]1062; AVX2-NEXT:    vpshufb %ymm13, %ymm0, %ymm91063; AVX2-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0],ymm8[1],ymm9[2,3,4],ymm8[5],ymm9[6,7]1064; AVX2-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1],ymm6[2,3],ymm8[4,5],ymm6[6,7]1065; AVX2-NEXT:    vpshufb %ymm11, %ymm3, %ymm31066; AVX2-NEXT:    vpshufb %ymm14, %ymm4, %ymm41067; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3],ymm4[4,5,6],ymm3[7]1068; AVX2-NEXT:    vpshufb %ymm7, %ymm1, %ymm11069; AVX2-NEXT:    vpshufb %ymm15, %ymm0, %ymm01070; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4],ymm1[5],ymm0[6,7]1071; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2,3],ymm0[4,5],ymm3[6,7]1072; AVX2-NEXT:    vmovdqa %ymm0, 96(%rax)1073; AVX2-NEXT:    vmovdqa %ymm6, 64(%rax)1074; AVX2-NEXT:    vmovdqa %ymm5, 32(%rax)1075; AVX2-NEXT:    vmovdqa %ymm2, (%rax)1076; AVX2-NEXT:    vzeroupper1077; AVX2-NEXT:    retq1078;1079; AVX2-FP-LABEL: store_i16_stride8_vf8:1080; AVX2-FP:       # %bb.0:1081; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %rax1082; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %r101083; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %r111084; AVX2-FP-NEXT:    vmovdqa (%rdi), %xmm01085; AVX2-FP-NEXT:    vmovdqa (%rdx), %xmm11086; AVX2-FP-NEXT:    vmovdqa (%r8), %xmm21087; AVX2-FP-NEXT:    vmovdqa (%r11), %xmm31088; AVX2-FP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm01089; AVX2-FP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm11090; AVX2-FP-NEXT:    vinserti128 $1, (%r9), %ymm2, %ymm41091; AVX2-FP-NEXT:    vinserti128 $1, (%r10), %ymm3, %ymm31092; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm5 = ymm3[0,2,0,2]1093; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm6 = [u,u,u,u,u,u,u,u,u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,u,u,u,u,2,3,10,11]1094; AVX2-FP-NEXT:    vpshufb %ymm6, %ymm5, %ymm21095; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm7 = ymm4[0,2,0,2]1096; AVX2-FP-NEXT:    vpmovsxdq {{.*#+}} ymm8 = [0,151519488,0,185205506]1097; AVX2-FP-NEXT:    vpshufb %ymm8, %ymm7, %ymm91098; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm9[0,1,2],ymm2[3],ymm9[4,5,6],ymm2[7]1099; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm9 = ymm1[0,2,0,2]1100; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm10 = [u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,u,u,u,u,2,3,10,11,u,u,u,u,u,u,u,u]1101; AVX2-FP-NEXT:    vpshufb %ymm10, %ymm9, %ymm111102; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm12 = ymm0[0,2,0,2]1103; AVX2-FP-NEXT:    vpmovsxdq {{.*#+}} ymm13 = [151519488,0,185205506,0]1104; AVX2-FP-NEXT:    vpshufb %ymm13, %ymm12, %ymm141105; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm11 = ymm14[0],ymm11[1],ymm14[2,3,4],ymm11[5],ymm14[6,7]1106; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm11[0,1],ymm2[2,3],ymm11[4,5],ymm2[6,7]1107; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm11 = [u,u,u,u,u,u,u,u,u,u,u,u,4,5,12,13,u,u,u,u,u,u,u,u,u,u,u,u,6,7,14,15]1108; AVX2-FP-NEXT:    vpshufb %ymm11, %ymm5, %ymm51109; AVX2-FP-NEXT:    vpmovsxdq {{.*#+}} ymm14 = [0,218891524,0,252577542]1110; AVX2-FP-NEXT:    vpshufb %ymm14, %ymm7, %ymm71111; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm7[0,1,2],ymm5[3],ymm7[4,5,6],ymm5[7]1112; AVX2-FP-NEXT:    vmovdqa {{.*#+}} ymm7 = [u,u,u,u,4,5,12,13,u,u,u,u,u,u,u,u,u,u,u,u,6,7,14,15,u,u,u,u,u,u,u,u]1113; AVX2-FP-NEXT:    vpshufb %ymm7, %ymm9, %ymm91114; AVX2-FP-NEXT:    vpmovsxdq {{.*#+}} ymm15 = [218891524,0,252577542,0]1115; AVX2-FP-NEXT:    vpshufb %ymm15, %ymm12, %ymm121116; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0],ymm9[1],ymm12[2,3,4],ymm9[5],ymm12[6,7]1117; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1],ymm5[2,3],ymm9[4,5],ymm5[6,7]1118; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[1,3,1,3]1119; AVX2-FP-NEXT:    vpshufb %ymm6, %ymm3, %ymm61120; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[1,3,1,3]1121; AVX2-FP-NEXT:    vpshufb %ymm8, %ymm4, %ymm81122; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2],ymm6[3],ymm8[4,5,6],ymm6[7]1123; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]1124; AVX2-FP-NEXT:    vpshufb %ymm10, %ymm1, %ymm81125; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,3,1,3]1126; AVX2-FP-NEXT:    vpshufb %ymm13, %ymm0, %ymm91127; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0],ymm8[1],ymm9[2,3,4],ymm8[5],ymm9[6,7]1128; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1],ymm6[2,3],ymm8[4,5],ymm6[6,7]1129; AVX2-FP-NEXT:    vpshufb %ymm11, %ymm3, %ymm31130; AVX2-FP-NEXT:    vpshufb %ymm14, %ymm4, %ymm41131; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3],ymm4[4,5,6],ymm3[7]1132; AVX2-FP-NEXT:    vpshufb %ymm7, %ymm1, %ymm11133; AVX2-FP-NEXT:    vpshufb %ymm15, %ymm0, %ymm01134; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4],ymm1[5],ymm0[6,7]1135; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2,3],ymm0[4,5],ymm3[6,7]1136; AVX2-FP-NEXT:    vmovdqa %ymm0, 96(%rax)1137; AVX2-FP-NEXT:    vmovdqa %ymm6, 64(%rax)1138; AVX2-FP-NEXT:    vmovdqa %ymm5, 32(%rax)1139; AVX2-FP-NEXT:    vmovdqa %ymm2, (%rax)1140; AVX2-FP-NEXT:    vzeroupper1141; AVX2-FP-NEXT:    retq1142;1143; AVX2-FCP-LABEL: store_i16_stride8_vf8:1144; AVX2-FCP:       # %bb.0:1145; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax1146; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r101147; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r111148; AVX2-FCP-NEXT:    vmovdqa (%rdi), %xmm01149; AVX2-FCP-NEXT:    vmovdqa (%rdx), %xmm11150; AVX2-FCP-NEXT:    vmovdqa (%r8), %xmm21151; AVX2-FCP-NEXT:    vmovdqa (%r11), %xmm31152; AVX2-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm01153; AVX2-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm11154; AVX2-FCP-NEXT:    vinserti128 $1, (%r9), %ymm2, %ymm41155; AVX2-FCP-NEXT:    vinserti128 $1, (%r10), %ymm3, %ymm31156; AVX2-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm3[0,2,0,2]1157; AVX2-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [u,u,u,u,u,u,u,u,u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,u,u,u,u,2,3,10,11]1158; AVX2-FCP-NEXT:    vpshufb %ymm6, %ymm5, %ymm21159; AVX2-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm4[0,2,0,2]1160; AVX2-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm8 = [0,151519488,0,185205506]1161; AVX2-FCP-NEXT:    vpshufb %ymm8, %ymm7, %ymm91162; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm9[0,1,2],ymm2[3],ymm9[4,5,6],ymm2[7]1163; AVX2-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm1[0,2,0,2]1164; AVX2-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,u,u,u,u,2,3,10,11,u,u,u,u,u,u,u,u]1165; AVX2-FCP-NEXT:    vpshufb %ymm10, %ymm9, %ymm111166; AVX2-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm0[0,2,0,2]1167; AVX2-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm13 = [151519488,0,185205506,0]1168; AVX2-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm141169; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm11 = ymm14[0],ymm11[1],ymm14[2,3,4],ymm11[5],ymm14[6,7]1170; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm11[0,1],ymm2[2,3],ymm11[4,5],ymm2[6,7]1171; AVX2-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [u,u,u,u,u,u,u,u,u,u,u,u,4,5,12,13,u,u,u,u,u,u,u,u,u,u,u,u,6,7,14,15]1172; AVX2-FCP-NEXT:    vpshufb %ymm11, %ymm5, %ymm51173; AVX2-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm14 = [0,218891524,0,252577542]1174; AVX2-FCP-NEXT:    vpshufb %ymm14, %ymm7, %ymm71175; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm7[0,1,2],ymm5[3],ymm7[4,5,6],ymm5[7]1176; AVX2-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [u,u,u,u,4,5,12,13,u,u,u,u,u,u,u,u,u,u,u,u,6,7,14,15,u,u,u,u,u,u,u,u]1177; AVX2-FCP-NEXT:    vpshufb %ymm7, %ymm9, %ymm91178; AVX2-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm15 = [218891524,0,252577542,0]1179; AVX2-FCP-NEXT:    vpshufb %ymm15, %ymm12, %ymm121180; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0],ymm9[1],ymm12[2,3,4],ymm9[5],ymm12[6,7]1181; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1],ymm5[2,3],ymm9[4,5],ymm5[6,7]1182; AVX2-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[1,3,1,3]1183; AVX2-FCP-NEXT:    vpshufb %ymm6, %ymm3, %ymm61184; AVX2-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[1,3,1,3]1185; AVX2-FCP-NEXT:    vpshufb %ymm8, %ymm4, %ymm81186; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2],ymm6[3],ymm8[4,5,6],ymm6[7]1187; AVX2-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]1188; AVX2-FCP-NEXT:    vpshufb %ymm10, %ymm1, %ymm81189; AVX2-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[1,3,1,3]1190; AVX2-FCP-NEXT:    vpshufb %ymm13, %ymm0, %ymm91191; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0],ymm8[1],ymm9[2,3,4],ymm8[5],ymm9[6,7]1192; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1],ymm6[2,3],ymm8[4,5],ymm6[6,7]1193; AVX2-FCP-NEXT:    vpshufb %ymm11, %ymm3, %ymm31194; AVX2-FCP-NEXT:    vpshufb %ymm14, %ymm4, %ymm41195; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3],ymm4[4,5,6],ymm3[7]1196; AVX2-FCP-NEXT:    vpshufb %ymm7, %ymm1, %ymm11197; AVX2-FCP-NEXT:    vpshufb %ymm15, %ymm0, %ymm01198; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4],ymm1[5],ymm0[6,7]1199; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm3[2,3],ymm0[4,5],ymm3[6,7]1200; AVX2-FCP-NEXT:    vmovdqa %ymm0, 96(%rax)1201; AVX2-FCP-NEXT:    vmovdqa %ymm6, 64(%rax)1202; AVX2-FCP-NEXT:    vmovdqa %ymm5, 32(%rax)1203; AVX2-FCP-NEXT:    vmovdqa %ymm2, (%rax)1204; AVX2-FCP-NEXT:    vzeroupper1205; AVX2-FCP-NEXT:    retq1206;1207; AVX512-LABEL: store_i16_stride8_vf8:1208; AVX512:       # %bb.0:1209; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax1210; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r101211; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r111212; AVX512-NEXT:    vmovdqa (%rdi), %xmm11213; AVX512-NEXT:    vmovdqa (%rdx), %xmm21214; AVX512-NEXT:    vmovdqa (%r8), %xmm31215; AVX512-NEXT:    vmovdqa (%r11), %xmm01216; AVX512-NEXT:    vinserti128 $1, (%r10), %ymm0, %ymm41217; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm4[0,2,0,2]1218; AVX512-NEXT:    vmovdqa {{.*#+}} ymm0 = [u,u,u,u,u,u,u,u,u,u,u,u,4,5,12,13,u,u,u,u,u,u,u,u,u,u,u,u,6,7,14,15]1219; AVX512-NEXT:    vpshufb %ymm0, %ymm5, %ymm61220; AVX512-NEXT:    vinserti128 $1, (%r9), %ymm3, %ymm31221; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm3[0,2,0,2]1222; AVX512-NEXT:    vpmovsxdq {{.*#+}} ymm8 = [0,218891524,0,252577542]1223; AVX512-NEXT:    vpshufb %ymm8, %ymm7, %ymm91224; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm9[0,1,2],ymm6[3],ymm9[4,5,6],ymm6[7]1225; AVX512-NEXT:    vinserti128 $1, (%rcx), %ymm2, %ymm21226; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm2[0,2,0,2]1227; AVX512-NEXT:    vmovdqa {{.*#+}} ymm10 = [u,u,u,u,4,5,12,13,u,u,u,u,u,u,u,u,u,u,u,u,6,7,14,15,u,u,u,u,u,u,u,u]1228; AVX512-NEXT:    vpshufb %ymm10, %ymm9, %ymm111229; AVX512-NEXT:    vinserti128 $1, (%rsi), %ymm1, %ymm11230; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm1[0,2,0,2]1231; AVX512-NEXT:    vpmovsxdq {{.*#+}} ymm13 = [218891524,0,252577542,0]1232; AVX512-NEXT:    vpshufb %ymm13, %ymm12, %ymm141233; AVX512-NEXT:    vpblendd {{.*#+}} ymm11 = ymm14[0],ymm11[1],ymm14[2,3,4],ymm11[5],ymm14[6,7]1234; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm11[0,1],ymm6[2,3],ymm11[4,5],ymm6[6,7]1235; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [u,u,u,u,u,u,u,u,u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,u,u,u,u,2,3,10,11]1236; AVX512-NEXT:    vpshufb %ymm11, %ymm5, %ymm51237; AVX512-NEXT:    vpmovsxdq {{.*#+}} ymm14 = [0,151519488,0,185205506]1238; AVX512-NEXT:    vpshufb %ymm14, %ymm7, %ymm71239; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm7[0,1,2],ymm5[3],ymm7[4,5,6],ymm5[7]1240; AVX512-NEXT:    vmovdqa {{.*#+}} ymm7 = [u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,u,u,u,u,2,3,10,11,u,u,u,u,u,u,u,u]1241; AVX512-NEXT:    vpshufb %ymm7, %ymm9, %ymm91242; AVX512-NEXT:    vpmovsxdq {{.*#+}} ymm15 = [151519488,0,185205506,0]1243; AVX512-NEXT:    vpshufb %ymm15, %ymm12, %ymm121244; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0],ymm9[1],ymm12[2,3,4],ymm9[5],ymm12[6,7]1245; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1],ymm5[2,3],ymm9[4,5],ymm5[6,7]1246; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm51247; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[1,3,1,3]1248; AVX512-NEXT:    vpshufb %ymm0, %ymm4, %ymm01249; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[1,3,1,3]1250; AVX512-NEXT:    vpshufb %ymm8, %ymm3, %ymm61251; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1,2],ymm0[3],ymm6[4,5,6],ymm0[7]1252; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,3,1,3]1253; AVX512-NEXT:    vpshufb %ymm10, %ymm2, %ymm61254; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]1255; AVX512-NEXT:    vpshufb %ymm13, %ymm1, %ymm81256; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0],ymm6[1],ymm8[2,3,4],ymm6[5],ymm8[6,7]1257; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1],ymm0[2,3],ymm6[4,5],ymm0[6,7]1258; AVX512-NEXT:    vpshufb %ymm11, %ymm4, %ymm41259; AVX512-NEXT:    vpshufb %ymm14, %ymm3, %ymm31260; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3],ymm3[4,5,6],ymm4[7]1261; AVX512-NEXT:    vpshufb %ymm7, %ymm2, %ymm21262; AVX512-NEXT:    vpshufb %ymm15, %ymm1, %ymm11263; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3,4],ymm2[5],ymm1[6,7]1264; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm3[2,3],ymm1[4,5],ymm3[6,7]1265; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm01266; AVX512-NEXT:    vmovdqa64 %zmm0, 64(%rax)1267; AVX512-NEXT:    vmovdqa64 %zmm5, (%rax)1268; AVX512-NEXT:    vzeroupper1269; AVX512-NEXT:    retq1270;1271; AVX512-FCP-LABEL: store_i16_stride8_vf8:1272; AVX512-FCP:       # %bb.0:1273; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax1274; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r101275; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r111276; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm11277; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm21278; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm31279; AVX512-FCP-NEXT:    vmovdqa (%r11), %xmm01280; AVX512-FCP-NEXT:    vinserti128 $1, (%r10), %ymm0, %ymm41281; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm4[0,2,0,2]1282; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [u,u,u,u,u,u,u,u,u,u,u,u,4,5,12,13,u,u,u,u,u,u,u,u,u,u,u,u,6,7,14,15]1283; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm5, %ymm61284; AVX512-FCP-NEXT:    vinserti128 $1, (%r9), %ymm3, %ymm31285; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm3[0,2,0,2]1286; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm8 = [0,218891524,0,252577542]1287; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm7, %ymm91288; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm9[0,1,2],ymm6[3],ymm9[4,5,6],ymm6[7]1289; AVX512-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm2, %ymm21290; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm2[0,2,0,2]1291; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [u,u,u,u,4,5,12,13,u,u,u,u,u,u,u,u,u,u,u,u,6,7,14,15,u,u,u,u,u,u,u,u]1292; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm9, %ymm111293; AVX512-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm1, %ymm11294; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm1[0,2,0,2]1295; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm13 = [218891524,0,252577542,0]1296; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm141297; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm11 = ymm14[0],ymm11[1],ymm14[2,3,4],ymm11[5],ymm14[6,7]1298; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm11[0,1],ymm6[2,3],ymm11[4,5],ymm6[6,7]1299; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [u,u,u,u,u,u,u,u,u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,u,u,u,u,2,3,10,11]1300; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm5, %ymm51301; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm14 = [0,151519488,0,185205506]1302; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm7, %ymm71303; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm7[0,1,2],ymm5[3],ymm7[4,5,6],ymm5[7]1304; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,u,u,u,u,2,3,10,11,u,u,u,u,u,u,u,u]1305; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm9, %ymm91306; AVX512-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm15 = [151519488,0,185205506,0]1307; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm12, %ymm121308; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0],ymm9[1],ymm12[2,3,4],ymm9[5],ymm12[6,7]1309; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1],ymm5[2,3],ymm9[4,5],ymm5[6,7]1310; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm51311; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[1,3,1,3]1312; AVX512-FCP-NEXT:    vpshufb %ymm0, %ymm4, %ymm01313; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[1,3,1,3]1314; AVX512-FCP-NEXT:    vpshufb %ymm8, %ymm3, %ymm61315; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1,2],ymm0[3],ymm6[4,5,6],ymm0[7]1316; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,3,1,3]1317; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm2, %ymm61318; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]1319; AVX512-FCP-NEXT:    vpshufb %ymm13, %ymm1, %ymm81320; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0],ymm6[1],ymm8[2,3,4],ymm6[5],ymm8[6,7]1321; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1],ymm0[2,3],ymm6[4,5],ymm0[6,7]1322; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm4, %ymm41323; AVX512-FCP-NEXT:    vpshufb %ymm14, %ymm3, %ymm31324; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3],ymm3[4,5,6],ymm4[7]1325; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm2, %ymm21326; AVX512-FCP-NEXT:    vpshufb %ymm15, %ymm1, %ymm11327; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3,4],ymm2[5],ymm1[6,7]1328; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm3[2,3],ymm1[4,5],ymm3[6,7]1329; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm01330; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rax)1331; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, (%rax)1332; AVX512-FCP-NEXT:    vzeroupper1333; AVX512-FCP-NEXT:    retq1334;1335; AVX512DQ-LABEL: store_i16_stride8_vf8:1336; AVX512DQ:       # %bb.0:1337; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax1338; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %r101339; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %r111340; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm11341; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm21342; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm31343; AVX512DQ-NEXT:    vmovdqa (%r11), %xmm01344; AVX512DQ-NEXT:    vinserti128 $1, (%r10), %ymm0, %ymm41345; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm4[0,2,0,2]1346; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm0 = [u,u,u,u,u,u,u,u,u,u,u,u,4,5,12,13,u,u,u,u,u,u,u,u,u,u,u,u,6,7,14,15]1347; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm5, %ymm61348; AVX512DQ-NEXT:    vinserti128 $1, (%r9), %ymm3, %ymm31349; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm3[0,2,0,2]1350; AVX512DQ-NEXT:    vpmovsxdq {{.*#+}} ymm8 = [0,218891524,0,252577542]1351; AVX512DQ-NEXT:    vpshufb %ymm8, %ymm7, %ymm91352; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm6 = ymm9[0,1,2],ymm6[3],ymm9[4,5,6],ymm6[7]1353; AVX512DQ-NEXT:    vinserti128 $1, (%rcx), %ymm2, %ymm21354; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm2[0,2,0,2]1355; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm10 = [u,u,u,u,4,5,12,13,u,u,u,u,u,u,u,u,u,u,u,u,6,7,14,15,u,u,u,u,u,u,u,u]1356; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm9, %ymm111357; AVX512DQ-NEXT:    vinserti128 $1, (%rsi), %ymm1, %ymm11358; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm1[0,2,0,2]1359; AVX512DQ-NEXT:    vpmovsxdq {{.*#+}} ymm13 = [218891524,0,252577542,0]1360; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm12, %ymm141361; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm11 = ymm14[0],ymm11[1],ymm14[2,3,4],ymm11[5],ymm14[6,7]1362; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm6 = ymm11[0,1],ymm6[2,3],ymm11[4,5],ymm6[6,7]1363; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [u,u,u,u,u,u,u,u,u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,u,u,u,u,2,3,10,11]1364; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm5, %ymm51365; AVX512DQ-NEXT:    vpmovsxdq {{.*#+}} ymm14 = [0,151519488,0,185205506]1366; AVX512DQ-NEXT:    vpshufb %ymm14, %ymm7, %ymm71367; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm7[0,1,2],ymm5[3],ymm7[4,5,6],ymm5[7]1368; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm7 = [u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,u,u,u,u,2,3,10,11,u,u,u,u,u,u,u,u]1369; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm9, %ymm91370; AVX512DQ-NEXT:    vpmovsxdq {{.*#+}} ymm15 = [151519488,0,185205506,0]1371; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm12, %ymm121372; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0],ymm9[1],ymm12[2,3,4],ymm9[5],ymm12[6,7]1373; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1],ymm5[2,3],ymm9[4,5],ymm5[6,7]1374; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm51375; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[1,3,1,3]1376; AVX512DQ-NEXT:    vpshufb %ymm0, %ymm4, %ymm01377; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[1,3,1,3]1378; AVX512DQ-NEXT:    vpshufb %ymm8, %ymm3, %ymm61379; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1,2],ymm0[3],ymm6[4,5,6],ymm0[7]1380; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,3,1,3]1381; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm2, %ymm61382; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]1383; AVX512DQ-NEXT:    vpshufb %ymm13, %ymm1, %ymm81384; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0],ymm6[1],ymm8[2,3,4],ymm6[5],ymm8[6,7]1385; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1],ymm0[2,3],ymm6[4,5],ymm0[6,7]1386; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm4, %ymm41387; AVX512DQ-NEXT:    vpshufb %ymm14, %ymm3, %ymm31388; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3],ymm3[4,5,6],ymm4[7]1389; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm2, %ymm21390; AVX512DQ-NEXT:    vpshufb %ymm15, %ymm1, %ymm11391; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3,4],ymm2[5],ymm1[6,7]1392; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm3[2,3],ymm1[4,5],ymm3[6,7]1393; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm01394; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 64(%rax)1395; AVX512DQ-NEXT:    vmovdqa64 %zmm5, (%rax)1396; AVX512DQ-NEXT:    vzeroupper1397; AVX512DQ-NEXT:    retq1398;1399; AVX512DQ-FCP-LABEL: store_i16_stride8_vf8:1400; AVX512DQ-FCP:       # %bb.0:1401; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax1402; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r101403; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r111404; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm11405; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm21406; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm31407; AVX512DQ-FCP-NEXT:    vmovdqa (%r11), %xmm01408; AVX512DQ-FCP-NEXT:    vinserti128 $1, (%r10), %ymm0, %ymm41409; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm4[0,2,0,2]1410; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [u,u,u,u,u,u,u,u,u,u,u,u,4,5,12,13,u,u,u,u,u,u,u,u,u,u,u,u,6,7,14,15]1411; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm5, %ymm61412; AVX512DQ-FCP-NEXT:    vinserti128 $1, (%r9), %ymm3, %ymm31413; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm3[0,2,0,2]1414; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm8 = [0,218891524,0,252577542]1415; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm7, %ymm91416; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm9[0,1,2],ymm6[3],ymm9[4,5,6],ymm6[7]1417; AVX512DQ-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm2, %ymm21418; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm2[0,2,0,2]1419; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [u,u,u,u,4,5,12,13,u,u,u,u,u,u,u,u,u,u,u,u,6,7,14,15,u,u,u,u,u,u,u,u]1420; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm9, %ymm111421; AVX512DQ-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm1, %ymm11422; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm1[0,2,0,2]1423; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm13 = [218891524,0,252577542,0]1424; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm141425; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm11 = ymm14[0],ymm11[1],ymm14[2,3,4],ymm11[5],ymm14[6,7]1426; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm11[0,1],ymm6[2,3],ymm11[4,5],ymm6[6,7]1427; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [u,u,u,u,u,u,u,u,u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,u,u,u,u,2,3,10,11]1428; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm5, %ymm51429; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm14 = [0,151519488,0,185205506]1430; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm7, %ymm71431; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm7[0,1,2],ymm5[3],ymm7[4,5,6],ymm5[7]1432; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [u,u,u,u,0,1,8,9,u,u,u,u,u,u,u,u,u,u,u,u,2,3,10,11,u,u,u,u,u,u,u,u]1433; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm9, %ymm91434; AVX512DQ-FCP-NEXT:    vpmovsxdq {{.*#+}} ymm15 = [151519488,0,185205506,0]1435; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm12, %ymm121436; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0],ymm9[1],ymm12[2,3,4],ymm9[5],ymm12[6,7]1437; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1],ymm5[2,3],ymm9[4,5],ymm5[6,7]1438; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm5, %zmm51439; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[1,3,1,3]1440; AVX512DQ-FCP-NEXT:    vpshufb %ymm0, %ymm4, %ymm01441; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[1,3,1,3]1442; AVX512DQ-FCP-NEXT:    vpshufb %ymm8, %ymm3, %ymm61443; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1,2],ymm0[3],ymm6[4,5,6],ymm0[7]1444; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[1,3,1,3]1445; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm2, %ymm61446; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,3,1,3]1447; AVX512DQ-FCP-NEXT:    vpshufb %ymm13, %ymm1, %ymm81448; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0],ymm6[1],ymm8[2,3,4],ymm6[5],ymm8[6,7]1449; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1],ymm0[2,3],ymm6[4,5],ymm0[6,7]1450; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm4, %ymm41451; AVX512DQ-FCP-NEXT:    vpshufb %ymm14, %ymm3, %ymm31452; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3],ymm3[4,5,6],ymm4[7]1453; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm2, %ymm21454; AVX512DQ-FCP-NEXT:    vpshufb %ymm15, %ymm1, %ymm11455; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3,4],ymm2[5],ymm1[6,7]1456; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm3[2,3],ymm1[4,5],ymm3[6,7]1457; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm01458; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rax)1459; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, (%rax)1460; AVX512DQ-FCP-NEXT:    vzeroupper1461; AVX512DQ-FCP-NEXT:    retq1462;1463; AVX512BW-LABEL: store_i16_stride8_vf8:1464; AVX512BW:       # %bb.0:1465; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax1466; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r101467; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r111468; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm01469; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm11470; AVX512BW-NEXT:    vmovdqa (%r8), %xmm21471; AVX512BW-NEXT:    vmovdqa (%r11), %xmm31472; AVX512BW-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm11473; AVX512BW-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm01474; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm01475; AVX512BW-NEXT:    vinserti128 $1, (%r10), %ymm3, %ymm11476; AVX512BW-NEXT:    vinserti128 $1, (%r9), %ymm2, %ymm21477; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm11478; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,8,16,24,32,40,48,56,1,9,17,25,33,41,49,57,2,10,18,26,34,42,50,58,3,11,19,27,35,43,51,59]1479; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm21480; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [4,12,20,28,36,44,52,60,5,13,21,29,37,45,53,61,6,14,22,30,38,46,54,62,7,15,23,31,39,47,55,63]1481; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm31482; AVX512BW-NEXT:    vmovdqa64 %zmm3, 64(%rax)1483; AVX512BW-NEXT:    vmovdqa64 %zmm2, (%rax)1484; AVX512BW-NEXT:    vzeroupper1485; AVX512BW-NEXT:    retq1486;1487; AVX512BW-FCP-LABEL: store_i16_stride8_vf8:1488; AVX512BW-FCP:       # %bb.0:1489; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax1490; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r101491; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r111492; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %xmm01493; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %xmm11494; AVX512BW-FCP-NEXT:    vmovdqa (%r8), %xmm21495; AVX512BW-FCP-NEXT:    vmovdqa (%r11), %xmm31496; AVX512BW-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm11497; AVX512BW-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm01498; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm01499; AVX512BW-FCP-NEXT:    vinserti128 $1, (%r10), %ymm3, %ymm11500; AVX512BW-FCP-NEXT:    vinserti128 $1, (%r9), %ymm2, %ymm21501; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm11502; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,8,16,24,32,40,48,56,1,9,17,25,33,41,49,57,2,10,18,26,34,42,50,58,3,11,19,27,35,43,51,59]1503; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm21504; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [4,12,20,28,36,44,52,60,5,13,21,29,37,45,53,61,6,14,22,30,38,46,54,62,7,15,23,31,39,47,55,63]1505; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm31506; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, 64(%rax)1507; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm2, (%rax)1508; AVX512BW-FCP-NEXT:    vzeroupper1509; AVX512BW-FCP-NEXT:    retq1510;1511; AVX512DQ-BW-LABEL: store_i16_stride8_vf8:1512; AVX512DQ-BW:       # %bb.0:1513; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax1514; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %r101515; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %r111516; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %xmm01517; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %xmm11518; AVX512DQ-BW-NEXT:    vmovdqa (%r8), %xmm21519; AVX512DQ-BW-NEXT:    vmovdqa (%r11), %xmm31520; AVX512DQ-BW-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm11521; AVX512DQ-BW-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm01522; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm01523; AVX512DQ-BW-NEXT:    vinserti128 $1, (%r10), %ymm3, %ymm11524; AVX512DQ-BW-NEXT:    vinserti128 $1, (%r9), %ymm2, %ymm21525; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm11526; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,8,16,24,32,40,48,56,1,9,17,25,33,41,49,57,2,10,18,26,34,42,50,58,3,11,19,27,35,43,51,59]1527; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm21528; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [4,12,20,28,36,44,52,60,5,13,21,29,37,45,53,61,6,14,22,30,38,46,54,62,7,15,23,31,39,47,55,63]1529; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm31530; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm3, 64(%rax)1531; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm2, (%rax)1532; AVX512DQ-BW-NEXT:    vzeroupper1533; AVX512DQ-BW-NEXT:    retq1534;1535; AVX512DQ-BW-FCP-LABEL: store_i16_stride8_vf8:1536; AVX512DQ-BW-FCP:       # %bb.0:1537; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax1538; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r101539; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r111540; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %xmm01541; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %xmm11542; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r8), %xmm21543; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r11), %xmm31544; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm11545; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, (%rsi), %ymm0, %ymm01546; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm01547; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, (%r10), %ymm3, %ymm11548; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, (%r9), %ymm2, %ymm21549; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm11550; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,8,16,24,32,40,48,56,1,9,17,25,33,41,49,57,2,10,18,26,34,42,50,58,3,11,19,27,35,43,51,59]1551; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm21552; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [4,12,20,28,36,44,52,60,5,13,21,29,37,45,53,61,6,14,22,30,38,46,54,62,7,15,23,31,39,47,55,63]1553; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm31554; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm3, 64(%rax)1555; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm2, (%rax)1556; AVX512DQ-BW-FCP-NEXT:    vzeroupper1557; AVX512DQ-BW-FCP-NEXT:    retq1558  %in.vec0 = load <8 x i16>, ptr %in.vecptr0, align 641559  %in.vec1 = load <8 x i16>, ptr %in.vecptr1, align 641560  %in.vec2 = load <8 x i16>, ptr %in.vecptr2, align 641561  %in.vec3 = load <8 x i16>, ptr %in.vecptr3, align 641562  %in.vec4 = load <8 x i16>, ptr %in.vecptr4, align 641563  %in.vec5 = load <8 x i16>, ptr %in.vecptr5, align 641564  %in.vec6 = load <8 x i16>, ptr %in.vecptr6, align 641565  %in.vec7 = load <8 x i16>, ptr %in.vecptr7, align 641566  %1 = shufflevector <8 x i16> %in.vec0, <8 x i16> %in.vec1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1567  %2 = shufflevector <8 x i16> %in.vec2, <8 x i16> %in.vec3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1568  %3 = shufflevector <8 x i16> %in.vec4, <8 x i16> %in.vec5, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1569  %4 = shufflevector <8 x i16> %in.vec6, <8 x i16> %in.vec7, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1570  %5 = shufflevector <16 x i16> %1, <16 x i16> %2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>1571  %6 = shufflevector <16 x i16> %3, <16 x i16> %4, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>1572  %7 = shufflevector <32 x i16> %5, <32 x i16> %6, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>1573  %interleaved.vec = shufflevector <64 x i16> %7, <64 x i16> poison, <64 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 40, i32 48, i32 56, i32 1, i32 9, i32 17, i32 25, i32 33, i32 41, i32 49, i32 57, i32 2, i32 10, i32 18, i32 26, i32 34, i32 42, i32 50, i32 58, i32 3, i32 11, i32 19, i32 27, i32 35, i32 43, i32 51, i32 59, i32 4, i32 12, i32 20, i32 28, i32 36, i32 44, i32 52, i32 60, i32 5, i32 13, i32 21, i32 29, i32 37, i32 45, i32 53, i32 61, i32 6, i32 14, i32 22, i32 30, i32 38, i32 46, i32 54, i32 62, i32 7, i32 15, i32 23, i32 31, i32 39, i32 47, i32 55, i32 63>1574  store <64 x i16> %interleaved.vec, ptr %out.vec, align 641575  ret void1576}1577 1578define void @store_i16_stride8_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %in.vecptr4, ptr %in.vecptr5, ptr %in.vecptr6, ptr %in.vecptr7, ptr %out.vec) nounwind {1579; SSE-LABEL: store_i16_stride8_vf16:1580; SSE:       # %bb.0:1581; SSE-NEXT:    subq $88, %rsp1582; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %rax1583; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %r101584; SSE-NEXT:    movdqa (%rdi), %xmm101585; SSE-NEXT:    movdqa 16(%rdi), %xmm151586; SSE-NEXT:    movdqa (%rsi), %xmm01587; SSE-NEXT:    movdqa (%rdx), %xmm71588; SSE-NEXT:    movdqa (%rcx), %xmm11589; SSE-NEXT:    movdqa (%r8), %xmm81590; SSE-NEXT:    movdqa (%r9), %xmm21591; SSE-NEXT:    movdqa (%r10), %xmm111592; SSE-NEXT:    movdqa (%rax), %xmm31593; SSE-NEXT:    movdqa %xmm11, %xmm51594; SSE-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]1595; SSE-NEXT:    movdqa %xmm8, %xmm121596; SSE-NEXT:    punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm2[0],xmm12[1],xmm2[1],xmm12[2],xmm2[2],xmm12[3],xmm2[3]1597; SSE-NEXT:    movdqa %xmm12, %xmm41598; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]1599; SSE-NEXT:    movdqa %xmm5, %xmm141600; SSE-NEXT:    movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1601; SSE-NEXT:    movdqa %xmm7, %xmm61602; SSE-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3]1603; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,1,1]1604; SSE-NEXT:    movdqa %xmm6, %xmm91605; SSE-NEXT:    movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1606; SSE-NEXT:    movdqa %xmm10, %xmm131607; SSE-NEXT:    punpcklwd {{.*#+}} xmm13 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3]1608; SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm13[1,1,1,1]1609; SSE-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]1610; SSE-NEXT:    shufps {{.*#+}} xmm6 = xmm6[0,1],xmm4[2,3]1611; SSE-NEXT:    movaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1612; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm14[0,0,0,0]1613; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[0,0,0,0]1614; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]1615; SSE-NEXT:    movdqa %xmm13, %xmm41616; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm9[0],xmm4[1],xmm9[1]1617; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm4[0],xmm5[1]1618; SSE-NEXT:    movapd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1619; SSE-NEXT:    movdqa 16(%rdx), %xmm91620; SSE-NEXT:    punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm3[4],xmm11[5],xmm3[5],xmm11[6],xmm3[6],xmm11[7],xmm3[7]1621; SSE-NEXT:    movdqa 16(%rcx), %xmm31622; SSE-NEXT:    movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1623; SSE-NEXT:    punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm2[4],xmm8[5],xmm2[5],xmm8[6],xmm2[6],xmm8[7],xmm2[7]1624; SSE-NEXT:    movdqa %xmm8, %xmm21625; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm11[0],xmm2[1],xmm11[1]1626; SSE-NEXT:    movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1627; SSE-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm1[4],xmm7[5],xmm1[5],xmm7[6],xmm1[6],xmm7[7],xmm1[7]1628; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm7[1,1,1,1]1629; SSE-NEXT:    movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1630; SSE-NEXT:    punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm0[4],xmm10[5],xmm0[5],xmm10[6],xmm0[6],xmm10[7],xmm0[7]1631; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[1,1,1,1]1632; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]1633; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]1634; SSE-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1635; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm11[0,0,0,0]1636; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm8[0,0,0,0]1637; SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]1638; SSE-NEXT:    movdqa %xmm10, %xmm01639; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1]1640; SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]1641; SSE-NEXT:    movapd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1642; SSE-NEXT:    movdqa %xmm9, %xmm51643; SSE-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]1644; SSE-NEXT:    movdqa 16(%rsi), %xmm01645; SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1646; SSE-NEXT:    movdqa %xmm15, %xmm61647; SSE-NEXT:    movdqa %xmm15, %xmm31648; SSE-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]1649; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,1,1]1650; SSE-NEXT:    movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1651; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]1652; SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]1653; SSE-NEXT:    movdqa 16(%r10), %xmm151654; SSE-NEXT:    movdqa 16(%rax), %xmm71655; SSE-NEXT:    movdqa %xmm15, %xmm141656; SSE-NEXT:    punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm7[0],xmm14[1],xmm7[1],xmm14[2],xmm7[2],xmm14[3],xmm7[3]1657; SSE-NEXT:    movdqa 16(%r8), %xmm41658; SSE-NEXT:    movdqa 16(%r9), %xmm111659; SSE-NEXT:    movdqa %xmm4, %xmm21660; SSE-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm11[0],xmm2[1],xmm11[1],xmm2[2],xmm11[2],xmm2[3],xmm11[3]1661; SSE-NEXT:    movdqa %xmm2, %xmm01662; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1]1663; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3]1664; SSE-NEXT:    movaps %xmm1, (%rsp) # 16-byte Spill1665; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm14[0,0,0,0]1666; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,0,0,0]1667; SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]1668; SSE-NEXT:    movdqa %xmm3, %xmm01669; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]1670; SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]1671; SSE-NEXT:    movapd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1672; SSE-NEXT:    punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm7[4],xmm15[5],xmm7[5],xmm15[6],xmm7[6],xmm15[7],xmm7[7]1673; SSE-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm11[4],xmm4[5],xmm11[5],xmm4[6],xmm11[6],xmm4[7],xmm11[7]1674; SSE-NEXT:    punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload1675; SSE-NEXT:    # xmm9 = xmm9[4],mem[4],xmm9[5],mem[5],xmm9[6],mem[6],xmm9[7],mem[7]1676; SSE-NEXT:    punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload1677; SSE-NEXT:    # xmm6 = xmm6[4],mem[4],xmm6[5],mem[5],xmm6[6],mem[6],xmm6[7],mem[7]1678; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[1,1,1,1]1679; SSE-NEXT:    movdqa %xmm9, %xmm11680; SSE-NEXT:    movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1681; SSE-NEXT:    pshufd {{.*#+}} xmm11 = xmm6[1,1,1,1]1682; SSE-NEXT:    movdqa %xmm6, %xmm71683; SSE-NEXT:    punpckldq {{.*#+}} xmm11 = xmm11[0],xmm0[0],xmm11[1],xmm0[1]1684; SSE-NEXT:    movdqa %xmm4, %xmm01685; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm15[0],xmm0[1],xmm15[1]1686; SSE-NEXT:    shufps {{.*#+}} xmm11 = xmm11[0,1],xmm0[2,3]1687; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm15[0,0,0,0]1688; SSE-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[0,0,0,0]1689; SSE-NEXT:    punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm0[2],xmm9[3],xmm0[3]1690; SSE-NEXT:    movdqa %xmm6, %xmm01691; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]1692; SSE-NEXT:    movsd {{.*#+}} xmm9 = xmm0[0],xmm9[1]1693; SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1694; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]1695; SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm13[3,3,3,3]1696; SSE-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]1697; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[2,2,2,2]1698; SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload1699; SSE-NEXT:    punpckhdq {{.*#+}} xmm12 = xmm12[2],xmm1[2],xmm12[3],xmm1[3]1700; SSE-NEXT:    shufps {{.*#+}} xmm6 = xmm6[0,1],xmm12[2,3]1701; SSE-NEXT:    punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm0[2],xmm13[3],xmm0[3]1702; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm1[2,2,2,2]1703; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm12[2],xmm5[3],xmm12[3]1704; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm13[0],xmm5[1]1705; SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1706; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm0[3,3,3,3]1707; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm10[3,3,3,3]1708; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]1709; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm8[2,2,2,2]1710; SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload1711; SSE-NEXT:    punpckhdq {{.*#+}} xmm8 = xmm8[2],xmm1[2],xmm8[3],xmm1[3]1712; SSE-NEXT:    shufps {{.*#+}} xmm12 = xmm12[0,1],xmm8[2,3]1713; SSE-NEXT:    punpckhdq {{.*#+}} xmm10 = xmm10[2],xmm0[2],xmm10[3],xmm0[3]1714; SSE-NEXT:    pshufd {{.*#+}} xmm8 = xmm1[2,2,2,2]1715; SSE-NEXT:    punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm8[2],xmm13[3],xmm8[3]1716; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm10[0],xmm13[1]1717; SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1718; SSE-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[3,3,3,3]1719; SSE-NEXT:    pshufd {{.*#+}} xmm8 = xmm3[3,3,3,3]1720; SSE-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]1721; SSE-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[2,2,2,2]1722; SSE-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm14[2],xmm2[3],xmm14[3]1723; SSE-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,1],xmm2[2,3]1724; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm0[2],xmm3[3],xmm0[3]1725; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm14[2,2,2,2]1726; SSE-NEXT:    punpckhdq {{.*#+}} xmm10 = xmm10[2],xmm1[2],xmm10[3],xmm1[3]1727; SSE-NEXT:    movsd {{.*#+}} xmm10 = xmm3[0],xmm10[1]1728; SSE-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1729; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]1730; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[3,3,3,3]1731; SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]1732; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[2,2,2,2]1733; SSE-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm15[2],xmm4[3],xmm15[3]1734; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3]1735; SSE-NEXT:    punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm0[2],xmm7[3],xmm0[3]1736; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm15[2,2,2,2]1737; SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]1738; SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm7[0],xmm1[1]1739; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %rax1740; SSE-NEXT:    movapd %xmm1, 224(%rax)1741; SSE-NEXT:    movaps %xmm3, 240(%rax)1742; SSE-NEXT:    movapd %xmm10, 160(%rax)1743; SSE-NEXT:    movaps %xmm8, 176(%rax)1744; SSE-NEXT:    movapd %xmm13, 96(%rax)1745; SSE-NEXT:    movaps %xmm12, 112(%rax)1746; SSE-NEXT:    movapd %xmm5, 32(%rax)1747; SSE-NEXT:    movaps %xmm6, 48(%rax)1748; SSE-NEXT:    movapd %xmm9, 192(%rax)1749; SSE-NEXT:    movaps %xmm11, 208(%rax)1750; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1751; SSE-NEXT:    movaps %xmm0, 128(%rax)1752; SSE-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload1753; SSE-NEXT:    movaps %xmm0, 144(%rax)1754; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1755; SSE-NEXT:    movaps %xmm0, 64(%rax)1756; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1757; SSE-NEXT:    movaps %xmm0, 80(%rax)1758; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1759; SSE-NEXT:    movaps %xmm0, (%rax)1760; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload1761; SSE-NEXT:    movaps %xmm0, 16(%rax)1762; SSE-NEXT:    addq $88, %rsp1763; SSE-NEXT:    retq1764;1765; AVX-LABEL: store_i16_stride8_vf16:1766; AVX:       # %bb.0:1767; AVX-NEXT:    subq $136, %rsp1768; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax1769; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r101770; AVX-NEXT:    vmovdqa (%r10), %xmm31771; AVX-NEXT:    vmovaps 16(%r10), %xmm01772; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1773; AVX-NEXT:    vmovdqa (%rax), %xmm51774; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]1775; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]1776; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[0,1,0,1]1777; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm11778; AVX-NEXT:    vmovdqa (%r9), %xmm21779; AVX-NEXT:    vmovdqa (%r8), %xmm61780; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]1781; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm12[0,1,0,1]1782; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm12[0],zero,xmm12[1],zero1783; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm7, %ymm71784; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm7[0,1,2],ymm1[3],ymm7[4,5,6],ymm1[7]1785; AVX-NEXT:    vmovdqa (%rsi), %xmm41786; AVX-NEXT:    vmovdqa (%rdi), %xmm81787; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3]1788; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm14[1,1,1,1]1789; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm14, %ymm91790; AVX-NEXT:    vmovdqa (%rcx), %xmm101791; AVX-NEXT:    vmovdqa (%rdx), %xmm111792; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]1793; AVX-NEXT:    vpshufd {{.*#+}} xmm13 = xmm15[0,0,1,1]1794; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm13, %ymm131795; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm9[0],ymm13[1],ymm9[2,3,4],ymm13[5],ymm9[6,7]1796; AVX-NEXT:    vmovaps 16(%rax), %xmm71797; AVX-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1798; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm13[0,1],ymm1[2,3],ymm13[4,5],ymm1[6,7]1799; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1800; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm12[2,2,3,3]1801; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm12, %ymm11802; AVX-NEXT:    vpshufd {{.*#+}} xmm12 = xmm0[2,2,2,2]1803; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm12, %ymm01804; AVX-NEXT:    vmovdqa 16(%r9), %xmm91805; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6],ymm0[7]1806; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm15[2,2,3,3]1807; AVX-NEXT:    vpshufd {{.*#+}} xmm12 = xmm15[2,3,2,3]1808; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm0, %ymm121809; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm14[2,3,2,3]1810; AVX-NEXT:    vpshufd {{.*#+}} xmm14 = xmm14[3,3,3,3]1811; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm0, %ymm141812; AVX-NEXT:    vmovdqa 16(%r8), %xmm131813; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm14[0],ymm12[1],ymm14[2,3,4],ymm12[5],ymm14[6,7]1814; AVX-NEXT:    vmovdqa 16(%rcx), %xmm141815; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm12[0,1],ymm1[2,3],ymm12[4,5],ymm1[6,7]1816; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1817; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]1818; AVX-NEXT:    vmovdqu %ymm0, (%rsp) # 32-byte Spill1819; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm0[2,2,3,3]1820; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm121821; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]1822; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1823; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,2,2,2]1824; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm31825; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm12[0,1,2],ymm3[3],ymm12[4,5,6],ymm3[7]1826; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]1827; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill1828; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm0[2,2,3,3]1829; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm0[2,3,2,3]1830; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm10, %ymm101831; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm8[4],xmm4[4],xmm8[5],xmm4[5],xmm8[6],xmm4[6],xmm8[7],xmm4[7]1832; AVX-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1833; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm0[2,3,2,3]1834; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm0[3,3,3,3]1835; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm8, %ymm81836; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0],ymm10[1],ymm8[2,3,4],ymm10[5],ymm8[6,7]1837; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm8[0,1],ymm5[2,3],ymm8[4,5],ymm5[6,7]1838; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1839; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm13[0],xmm9[0],xmm13[1],xmm9[1],xmm13[2],xmm9[2],xmm13[3],xmm9[3]1840; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm10[2,2,3,3]1841; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm10, %ymm81842; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload1843; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload1844; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]1845; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm12[2,2,2,2]1846; AVX-NEXT:    vinsertf128 $1, %xmm12, %ymm11, %ymm111847; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm8[0,1,2],ymm11[3],ymm8[4,5,6],ymm11[7]1848; AVX-NEXT:    vmovdqa 16(%rdx), %xmm81849; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm8[0],xmm14[0],xmm8[1],xmm14[1],xmm8[2],xmm14[2],xmm8[3],xmm14[3]1850; AVX-NEXT:    vpshufd {{.*#+}} xmm15 = xmm11[2,2,3,3]1851; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm11[2,3,2,3]1852; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm15, %ymm41853; AVX-NEXT:    vmovdqa 16(%rsi), %xmm61854; AVX-NEXT:    vmovdqa 16(%rdi), %xmm11855; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3]1856; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm15[2,3,2,3]1857; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm15[3,3,3,3]1858; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm01859; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm4[1],ymm0[2,3,4],ymm4[5],ymm0[6,7]1860; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm7[2,3],ymm0[4,5],ymm7[6,7]1861; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1862; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm13[4],xmm9[4],xmm13[5],xmm9[5],xmm13[6],xmm9[6],xmm13[7],xmm9[7]1863; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]1864; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[2,2,3,3]1865; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm51866; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm2[2,2,2,2]1867; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm7, %ymm71868; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm7[3],ymm5[4,5,6],ymm7[7]1869; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm8[4],xmm14[4],xmm8[5],xmm14[5],xmm8[6],xmm14[6],xmm8[7],xmm14[7]1870; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm7[2,2,3,3]1871; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm7[2,3,2,3]1872; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm4, %ymm41873; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]1874; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]1875; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm1[3,3,3,3]1876; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm3, %ymm31877; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3,4],ymm4[5],ymm3[6,7]1878; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm3[0,1],ymm5[2,3],ymm3[4,5],ymm5[6,7]1879; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[0,0,0,0]1880; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]1881; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm21882; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[0,1,0,1]1883; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero1884; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm01885; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3],ymm0[4,5,6],ymm2[7]1886; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]1887; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm11888; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm7[0,0,1,1]1889; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm2, %ymm21890; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3,4],ymm2[5],ymm1[6,7]1891; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]1892; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm12[0,0,0,0]1893; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm12[0,1,0,1]1894; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm11895; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm10[0,1,0,1]1896; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm10[0],zero,xmm10[1],zero1897; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm21898; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]1899; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm15[1,1,1,1]1900; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm15, %ymm21901; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm11[0,0,1,1]1902; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm3, %ymm31903; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2,3,4],ymm3[5],ymm2[6,7]1904; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5],ymm1[6,7]1905; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload1906; AVX-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[0,0,0,0]1907; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[0,1,0,1]1908; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm21909; AVX-NEXT:    vmovdqu (%rsp), %ymm5 # 32-byte Reload1910; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm5[0,1,0,1]1911; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero1912; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm31913; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]1914; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload1915; AVX-NEXT:    vshufps {{.*#+}} xmm3 = xmm5[1,1,1,1]1916; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm5, %ymm31917; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload1918; AVX-NEXT:    vshufps {{.*#+}} xmm5 = xmm6[0,0,1,1]1919; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm51920; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm5[1],ymm3[2,3,4],ymm5[5],ymm3[6,7]1921; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7]1922; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax1923; AVX-NEXT:    vmovaps %ymm2, 64(%rax)1924; AVX-NEXT:    vmovaps %ymm1, 128(%rax)1925; AVX-NEXT:    vmovaps %ymm0, 192(%rax)1926; AVX-NEXT:    vmovaps %ymm4, 224(%rax)1927; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload1928; AVX-NEXT:    vmovaps %ymm0, 160(%rax)1929; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload1930; AVX-NEXT:    vmovaps %ymm0, 96(%rax)1931; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload1932; AVX-NEXT:    vmovaps %ymm0, 32(%rax)1933; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload1934; AVX-NEXT:    vmovaps %ymm0, (%rax)1935; AVX-NEXT:    addq $136, %rsp1936; AVX-NEXT:    vzeroupper1937; AVX-NEXT:    retq1938;1939; AVX2-LABEL: store_i16_stride8_vf16:1940; AVX2:       # %bb.0:1941; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %rax1942; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r101943; AVX2-NEXT:    vmovdqa (%rdi), %ymm01944; AVX2-NEXT:    vmovdqa (%rsi), %ymm11945; AVX2-NEXT:    vmovdqa (%rax), %xmm51946; AVX2-NEXT:    vmovdqa (%r10), %xmm81947; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]1948; AVX2-NEXT:    vmovdqa (%r9), %xmm101949; AVX2-NEXT:    vmovdqa (%r8), %xmm111950; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]1951; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]1952; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]1953; AVX2-NEXT:    vmovdqa (%rcx), %xmm121954; AVX2-NEXT:    vmovdqa (%rdx), %xmm131955; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]1956; AVX2-NEXT:    vmovdqa (%rsi), %xmm141957; AVX2-NEXT:    vmovdqa (%rdi), %xmm151958; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]1959; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm9[0],xmm7[0],xmm9[1],xmm7[1]1960; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,1,3]1961; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1],ymm2[2,3],ymm4[4,5],ymm2[6,7]1962; AVX2-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1963; AVX2-NEXT:    vmovdqa (%rdx), %ymm41964; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm6[2],xmm3[2],xmm6[3],xmm3[3]1965; AVX2-NEXT:    vmovdqa (%rcx), %ymm61966; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm9 = xmm9[2],xmm7[2],xmm9[3],xmm7[3]1967; AVX2-NEXT:    vmovdqa (%r8), %ymm71968; AVX2-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]1969; AVX2-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,1,3]1970; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm9[0,1],ymm3[2,3],ymm9[4,5],ymm3[6,7]1971; AVX2-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill1972; AVX2-NEXT:    vmovdqa (%r9), %ymm91973; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7]1974; AVX2-NEXT:    vmovdqa (%r10), %ymm51975; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]1976; AVX2-NEXT:    vmovdqa (%rax), %ymm101977; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm13 = xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7]1978; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm12 = xmm11[2],xmm8[2],xmm11[3],xmm8[3]1979; AVX2-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,0,2,1]1980; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]1981; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm15 = xmm14[2],xmm13[2],xmm14[3],xmm13[3]1982; AVX2-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,1,1,3]1983; AVX2-NEXT:    vpblendd {{.*#+}} ymm12 = ymm15[0,1],ymm12[2,3],ymm15[4,5],ymm12[6,7]1984; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm8 = xmm11[0],xmm8[0],xmm11[1],xmm8[1]1985; AVX2-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,2,1]1986; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm11 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]1987; AVX2-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,1,1,3]1988; AVX2-NEXT:    vpblendd {{.*#+}} ymm8 = ymm11[0,1],ymm8[2,3],ymm11[4,5],ymm8[6,7]1989; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm5[0],ymm10[0],ymm5[1],ymm10[1],ymm5[2],ymm10[2],ymm5[3],ymm10[3],ymm5[8],ymm10[8],ymm5[9],ymm10[9],ymm5[10],ymm10[10],ymm5[11],ymm10[11]1990; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm13 = ymm7[0],ymm9[0],ymm7[1],ymm9[1],ymm7[2],ymm9[2],ymm7[3],ymm9[3],ymm7[8],ymm9[8],ymm7[9],ymm9[9],ymm7[10],ymm9[10],ymm7[11],ymm9[11]1991; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm14 = ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[6],ymm11[6],ymm13[7],ymm11[7]1992; AVX2-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,2,2,3]1993; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm4[0],ymm6[0],ymm4[1],ymm6[1],ymm4[2],ymm6[2],ymm4[3],ymm6[3],ymm4[8],ymm6[8],ymm4[9],ymm6[9],ymm4[10],ymm6[10],ymm4[11],ymm6[11]1994; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]1995; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm3[2],ymm15[2],ymm3[3],ymm15[3],ymm3[6],ymm15[6],ymm3[7],ymm15[7]1996; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,1,3,3]1997; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm14[2,3],ymm2[4,5],ymm14[6,7]1998; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm10[4],ymm5[5],ymm10[5],ymm5[6],ymm10[6],ymm5[7],ymm10[7],ymm5[12],ymm10[12],ymm5[13],ymm10[13],ymm5[14],ymm10[14],ymm5[15],ymm10[15]1999; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm7 = ymm7[4],ymm9[4],ymm7[5],ymm9[5],ymm7[6],ymm9[6],ymm7[7],ymm9[7],ymm7[12],ymm9[12],ymm7[13],ymm9[13],ymm7[14],ymm9[14],ymm7[15],ymm9[15]2000; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm4[4],ymm6[4],ymm4[5],ymm6[5],ymm4[6],ymm6[6],ymm4[7],ymm6[7],ymm4[12],ymm6[12],ymm4[13],ymm6[13],ymm4[14],ymm6[14],ymm4[15],ymm6[15]2001; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm6 = ymm7[2],ymm5[2],ymm7[3],ymm5[3],ymm7[6],ymm5[6],ymm7[7],ymm5[7]2002; AVX2-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,2,2,3]2003; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]2004; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm9 = ymm1[2],ymm4[2],ymm1[3],ymm4[3],ymm1[6],ymm4[6],ymm1[7],ymm4[7]2005; AVX2-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,1,3,3]2006; AVX2-NEXT:    vpblendd {{.*#+}} ymm6 = ymm9[0,1],ymm6[2,3],ymm9[4,5],ymm6[6,7]2007; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm5 = ymm7[0],ymm5[0],ymm7[1],ymm5[1],ymm7[4],ymm5[4],ymm7[5],ymm5[5]2008; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[1],ymm4[1],ymm1[4],ymm4[4],ymm1[5],ymm4[5]2009; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm5[0,2,2,3]2010; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,3,3]2011; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm4[2,3],ymm1[4,5],ymm4[6,7]2012; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm4 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[4],ymm11[4],ymm13[5],ymm11[5]2013; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm3[0],ymm15[0],ymm3[1],ymm15[1],ymm3[4],ymm15[4],ymm3[5],ymm15[5]2014; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3]2015; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3]2016; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm4[2,3],ymm0[4,5],ymm4[6,7]2017; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %rax2018; AVX2-NEXT:    vmovdqa %ymm0, 128(%rax)2019; AVX2-NEXT:    vmovdqa %ymm1, 192(%rax)2020; AVX2-NEXT:    vmovdqa %ymm6, 224(%rax)2021; AVX2-NEXT:    vmovdqa %ymm2, 160(%rax)2022; AVX2-NEXT:    vmovdqa %ymm8, 64(%rax)2023; AVX2-NEXT:    vmovdqa %ymm12, 96(%rax)2024; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2025; AVX2-NEXT:    vmovaps %ymm0, 32(%rax)2026; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2027; AVX2-NEXT:    vmovaps %ymm0, (%rax)2028; AVX2-NEXT:    vzeroupper2029; AVX2-NEXT:    retq2030;2031; AVX2-FP-LABEL: store_i16_stride8_vf16:2032; AVX2-FP:       # %bb.0:2033; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %rax2034; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %r102035; AVX2-FP-NEXT:    vmovdqa (%rdi), %ymm02036; AVX2-FP-NEXT:    vmovdqa (%rsi), %ymm12037; AVX2-FP-NEXT:    vmovdqa (%rax), %xmm52038; AVX2-FP-NEXT:    vmovdqa (%r10), %xmm82039; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]2040; AVX2-FP-NEXT:    vmovdqa (%r9), %xmm102041; AVX2-FP-NEXT:    vmovdqa (%r8), %xmm112042; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]2043; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]2044; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]2045; AVX2-FP-NEXT:    vmovdqa (%rcx), %xmm122046; AVX2-FP-NEXT:    vmovdqa (%rdx), %xmm132047; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]2048; AVX2-FP-NEXT:    vmovdqa (%rsi), %xmm142049; AVX2-FP-NEXT:    vmovdqa (%rdi), %xmm152050; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]2051; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm9[0],xmm7[0],xmm9[1],xmm7[1]2052; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,1,3]2053; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1],ymm2[2,3],ymm4[4,5],ymm2[6,7]2054; AVX2-FP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2055; AVX2-FP-NEXT:    vmovdqa (%rdx), %ymm42056; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm6[2],xmm3[2],xmm6[3],xmm3[3]2057; AVX2-FP-NEXT:    vmovdqa (%rcx), %ymm62058; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm9 = xmm9[2],xmm7[2],xmm9[3],xmm7[3]2059; AVX2-FP-NEXT:    vmovdqa (%r8), %ymm72060; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]2061; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,1,3]2062; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm9[0,1],ymm3[2,3],ymm9[4,5],ymm3[6,7]2063; AVX2-FP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2064; AVX2-FP-NEXT:    vmovdqa (%r9), %ymm92065; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7]2066; AVX2-FP-NEXT:    vmovdqa (%r10), %ymm52067; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]2068; AVX2-FP-NEXT:    vmovdqa (%rax), %ymm102069; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm13 = xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7]2070; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm12 = xmm11[2],xmm8[2],xmm11[3],xmm8[3]2071; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,0,2,1]2072; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]2073; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm15 = xmm14[2],xmm13[2],xmm14[3],xmm13[3]2074; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,1,1,3]2075; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm12 = ymm15[0,1],ymm12[2,3],ymm15[4,5],ymm12[6,7]2076; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm8 = xmm11[0],xmm8[0],xmm11[1],xmm8[1]2077; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,2,1]2078; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm11 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]2079; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,1,1,3]2080; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm8 = ymm11[0,1],ymm8[2,3],ymm11[4,5],ymm8[6,7]2081; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm5[0],ymm10[0],ymm5[1],ymm10[1],ymm5[2],ymm10[2],ymm5[3],ymm10[3],ymm5[8],ymm10[8],ymm5[9],ymm10[9],ymm5[10],ymm10[10],ymm5[11],ymm10[11]2082; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm13 = ymm7[0],ymm9[0],ymm7[1],ymm9[1],ymm7[2],ymm9[2],ymm7[3],ymm9[3],ymm7[8],ymm9[8],ymm7[9],ymm9[9],ymm7[10],ymm9[10],ymm7[11],ymm9[11]2083; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm14 = ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[6],ymm11[6],ymm13[7],ymm11[7]2084; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,2,2,3]2085; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm4[0],ymm6[0],ymm4[1],ymm6[1],ymm4[2],ymm6[2],ymm4[3],ymm6[3],ymm4[8],ymm6[8],ymm4[9],ymm6[9],ymm4[10],ymm6[10],ymm4[11],ymm6[11]2086; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]2087; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm3[2],ymm15[2],ymm3[3],ymm15[3],ymm3[6],ymm15[6],ymm3[7],ymm15[7]2088; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,1,3,3]2089; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm14[2,3],ymm2[4,5],ymm14[6,7]2090; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm10[4],ymm5[5],ymm10[5],ymm5[6],ymm10[6],ymm5[7],ymm10[7],ymm5[12],ymm10[12],ymm5[13],ymm10[13],ymm5[14],ymm10[14],ymm5[15],ymm10[15]2091; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm7 = ymm7[4],ymm9[4],ymm7[5],ymm9[5],ymm7[6],ymm9[6],ymm7[7],ymm9[7],ymm7[12],ymm9[12],ymm7[13],ymm9[13],ymm7[14],ymm9[14],ymm7[15],ymm9[15]2092; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm4[4],ymm6[4],ymm4[5],ymm6[5],ymm4[6],ymm6[6],ymm4[7],ymm6[7],ymm4[12],ymm6[12],ymm4[13],ymm6[13],ymm4[14],ymm6[14],ymm4[15],ymm6[15]2093; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm6 = ymm7[2],ymm5[2],ymm7[3],ymm5[3],ymm7[6],ymm5[6],ymm7[7],ymm5[7]2094; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,2,2,3]2095; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]2096; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm9 = ymm1[2],ymm4[2],ymm1[3],ymm4[3],ymm1[6],ymm4[6],ymm1[7],ymm4[7]2097; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,1,3,3]2098; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm9[0,1],ymm6[2,3],ymm9[4,5],ymm6[6,7]2099; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm5 = ymm7[0],ymm5[0],ymm7[1],ymm5[1],ymm7[4],ymm5[4],ymm7[5],ymm5[5]2100; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[1],ymm4[1],ymm1[4],ymm4[4],ymm1[5],ymm4[5]2101; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm4 = ymm5[0,2,2,3]2102; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,3,3]2103; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm4[2,3],ymm1[4,5],ymm4[6,7]2104; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm4 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[4],ymm11[4],ymm13[5],ymm11[5]2105; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm3[0],ymm15[0],ymm3[1],ymm15[1],ymm3[4],ymm15[4],ymm3[5],ymm15[5]2106; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3]2107; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3]2108; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm4[2,3],ymm0[4,5],ymm4[6,7]2109; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %rax2110; AVX2-FP-NEXT:    vmovdqa %ymm0, 128(%rax)2111; AVX2-FP-NEXT:    vmovdqa %ymm1, 192(%rax)2112; AVX2-FP-NEXT:    vmovdqa %ymm6, 224(%rax)2113; AVX2-FP-NEXT:    vmovdqa %ymm2, 160(%rax)2114; AVX2-FP-NEXT:    vmovdqa %ymm8, 64(%rax)2115; AVX2-FP-NEXT:    vmovdqa %ymm12, 96(%rax)2116; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2117; AVX2-FP-NEXT:    vmovaps %ymm0, 32(%rax)2118; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2119; AVX2-FP-NEXT:    vmovaps %ymm0, (%rax)2120; AVX2-FP-NEXT:    vzeroupper2121; AVX2-FP-NEXT:    retq2122;2123; AVX2-FCP-LABEL: store_i16_stride8_vf16:2124; AVX2-FCP:       # %bb.0:2125; AVX2-FCP-NEXT:    pushq %rax2126; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax2127; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r102128; AVX2-FCP-NEXT:    vmovdqa (%rax), %xmm52129; AVX2-FCP-NEXT:    vmovdqa (%r10), %xmm42130; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]2131; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [0,0,0,0,0,0,1,1]2132; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm0, %ymm02133; AVX2-FCP-NEXT:    vmovdqa (%r9), %xmm62134; AVX2-FCP-NEXT:    vmovdqa (%r8), %xmm92135; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3]2136; AVX2-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm2 = [0,0,0,1]2137; AVX2-FCP-NEXT:    vpermd %ymm11, %ymm2, %ymm72138; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm7[0,1,2],ymm0[3],ymm7[4,5,6],ymm0[7]2139; AVX2-FCP-NEXT:    vmovdqa (%rcx), %xmm102140; AVX2-FCP-NEXT:    vmovdqa (%rdx), %xmm122141; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]2142; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [0,0,1,1,1,1,0,0]2143; AVX2-FCP-NEXT:    vpermd %ymm13, %ymm0, %ymm142144; AVX2-FCP-NEXT:    vmovdqa (%rsi), %xmm152145; AVX2-FCP-NEXT:    vmovdqa (%rdi), %xmm22146; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3]2147; AVX2-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm8 = [0,1,1,0]2148; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm8, %ymm72149; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0],ymm14[1],ymm7[2,3,4],ymm14[5],ymm7[6,7]2150; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm7[0,1],ymm3[2,3],ymm7[4,5],ymm3[6,7]2151; AVX2-FCP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2152; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm7 = [2,2,2,2,0,0,3,3]2153; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm7, %ymm12154; AVX2-FCP-NEXT:    vpermd %ymm11, %ymm7, %ymm32155; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5,6],ymm1[7]2156; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm11 = [2,2,3,3,3,3,0,0]2157; AVX2-FCP-NEXT:    vpermd %ymm13, %ymm11, %ymm32158; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm11, %ymm02159; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2,3,4],ymm3[5],ymm0[6,7]2160; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]2161; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2162; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]2163; AVX2-FCP-NEXT:    vmovdqa (%rdi), %ymm142164; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm9[4],xmm6[4],xmm9[5],xmm6[5],xmm9[6],xmm6[6],xmm9[7],xmm6[7]2165; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm7, %ymm52166; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm7, %ymm62167; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0,1,2],ymm5[3],ymm6[4,5,6],ymm5[7]2168; AVX2-FCP-NEXT:    vmovdqa (%rsi), %ymm62169; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm12[4],xmm10[4],xmm12[5],xmm10[5],xmm12[6],xmm10[6],xmm12[7],xmm10[7]2170; AVX2-FCP-NEXT:    vmovdqa (%r8), %ymm92171; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm2[4],xmm15[4],xmm2[5],xmm15[5],xmm2[6],xmm15[6],xmm2[7],xmm15[7]2172; AVX2-FCP-NEXT:    vpermd %ymm7, %ymm11, %ymm22173; AVX2-FCP-NEXT:    vpermd %ymm10, %ymm11, %ymm112174; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm11[0],ymm2[1],ymm11[2,3,4],ymm2[5],ymm11[6,7]2175; AVX2-FCP-NEXT:    vmovdqa (%r9), %ymm122176; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm5[2,3],ymm2[4,5],ymm5[6,7]2177; AVX2-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2178; AVX2-FCP-NEXT:    vmovdqa (%r10), %ymm132179; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [0,0,0,0,0,0,1,1]2180; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm1, %ymm02181; AVX2-FCP-NEXT:    vmovdqa (%rax), %ymm152182; AVX2-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm1 = [0,0,0,1]2183; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm1, %ymm42184; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2],ymm0[3],ymm4[4,5,6],ymm0[7]2185; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [0,0,1,1,1,1,0,0]2186; AVX2-FCP-NEXT:    vpermd %ymm7, %ymm1, %ymm42187; AVX2-FCP-NEXT:    vpermd %ymm10, %ymm8, %ymm52188; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2,3,4],ymm4[5],ymm5[6,7]2189; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1],ymm0[2,3],ymm4[4,5],ymm0[6,7]2190; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill2191; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm13[0],ymm15[0],ymm13[1],ymm15[1],ymm13[2],ymm15[2],ymm13[3],ymm15[3],ymm13[8],ymm15[8],ymm13[9],ymm15[9],ymm13[10],ymm15[10],ymm13[11],ymm15[11]2192; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [0,2,4,6,4,6,6,7]2193; AVX2-FCP-NEXT:    vpermd %ymm5, %ymm0, %ymm82194; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm9[0],ymm12[0],ymm9[1],ymm12[1],ymm9[2],ymm12[2],ymm9[3],ymm12[3],ymm9[8],ymm12[8],ymm9[9],ymm12[9],ymm9[10],ymm12[10],ymm9[11],ymm12[11]2195; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [2,1,6,5,6,5,7,7]2196; AVX2-FCP-NEXT:    vpermd %ymm7, %ymm0, %ymm102197; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm8 = ymm10[0,1,2],ymm8[3],ymm10[4,5,6],ymm8[7]2198; AVX2-FCP-NEXT:    vmovdqa (%rdx), %ymm12199; AVX2-FCP-NEXT:    vmovdqa (%rcx), %ymm02200; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]2201; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm2 = [4,6,2,3,6,7,6,7]2202; AVX2-FCP-NEXT:    vpermd %ymm10, %ymm2, %ymm22203; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm14[0],ymm6[0],ymm14[1],ymm6[1],ymm14[2],ymm6[2],ymm14[3],ymm6[3],ymm14[8],ymm6[8],ymm14[9],ymm6[9],ymm14[10],ymm6[10],ymm14[11],ymm6[11]2204; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [6,5,3,3,7,7,7,7]2205; AVX2-FCP-NEXT:    vpermd %ymm11, %ymm3, %ymm42206; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2,3,4],ymm2[5],ymm4[6,7]2207; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm8 = ymm2[0,1],ymm8[2,3],ymm2[4,5],ymm8[6,7]2208; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm15[4],ymm13[5],ymm15[5],ymm13[6],ymm15[6],ymm13[7],ymm15[7],ymm13[12],ymm15[12],ymm13[13],ymm15[13],ymm13[14],ymm15[14],ymm13[15],ymm15[15]2209; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm9[4],ymm12[4],ymm9[5],ymm12[5],ymm9[6],ymm12[6],ymm9[7],ymm12[7],ymm9[12],ymm12[12],ymm9[13],ymm12[13],ymm9[14],ymm12[14],ymm9[15],ymm12[15]2210; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm9 = [0,2,4,6,4,6,6,7]2211; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm9, %ymm92212; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm12 = [2,1,6,5,6,5,7,7]2213; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm12, %ymm122214; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0,1,2],ymm9[3],ymm12[4,5,6],ymm9[7]2215; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]2216; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm14[4],ymm6[4],ymm14[5],ymm6[5],ymm14[6],ymm6[6],ymm14[7],ymm6[7],ymm14[12],ymm6[12],ymm14[13],ymm6[13],ymm14[14],ymm6[14],ymm14[15],ymm6[15]2217; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm6 = [4,6,2,3,6,7,6,7]2218; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm6, %ymm62219; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm32220; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0],ymm6[1],ymm3[2,3,4],ymm6[5],ymm3[6,7]2221; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm9[2,3],ymm3[4,5],ymm9[6,7]2222; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm6 = [0,0,4,4,4,4,6,5]2223; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm6, %ymm22224; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm9 = [0,1,4,5,4,5,5,7]2225; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm9, %ymm42226; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3],ymm4[4,5,6],ymm2[7]2227; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm4 = [4,4,2,1,6,5,6,5]2228; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm4, %ymm02229; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm12 = [4,5,1,3,5,7,5,7]2230; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm12, %ymm12231; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3,4],ymm0[5],ymm1[6,7]2232; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2,3],ymm0[4,5],ymm2[6,7]2233; AVX2-FCP-NEXT:    vpermd %ymm5, %ymm6, %ymm12234; AVX2-FCP-NEXT:    vpermd %ymm7, %ymm9, %ymm22235; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]2236; AVX2-FCP-NEXT:    vpermd %ymm10, %ymm4, %ymm22237; AVX2-FCP-NEXT:    vpermd %ymm11, %ymm12, %ymm42238; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2,3,4],ymm2[5],ymm4[6,7]2239; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5],ymm1[6,7]2240; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax2241; AVX2-FCP-NEXT:    vmovdqa %ymm1, 128(%rax)2242; AVX2-FCP-NEXT:    vmovdqa %ymm0, 192(%rax)2243; AVX2-FCP-NEXT:    vmovdqa %ymm3, 224(%rax)2244; AVX2-FCP-NEXT:    vmovdqa %ymm8, 160(%rax)2245; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2246; AVX2-FCP-NEXT:    vmovaps %ymm0, 64(%rax)2247; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2248; AVX2-FCP-NEXT:    vmovaps %ymm0, 96(%rax)2249; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2250; AVX2-FCP-NEXT:    vmovaps %ymm0, 32(%rax)2251; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload2252; AVX2-FCP-NEXT:    vmovaps %ymm0, (%rax)2253; AVX2-FCP-NEXT:    popq %rax2254; AVX2-FCP-NEXT:    vzeroupper2255; AVX2-FCP-NEXT:    retq2256;2257; AVX512-LABEL: store_i16_stride8_vf16:2258; AVX512:       # %bb.0:2259; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax2260; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r102261; AVX512-NEXT:    vmovdqa64 (%rdi), %ymm202262; AVX512-NEXT:    vmovdqa (%r8), %ymm92263; AVX512-NEXT:    vmovdqa (%r9), %ymm112264; AVX512-NEXT:    vmovdqa (%r10), %ymm122265; AVX512-NEXT:    vmovdqa (%rax), %ymm132266; AVX512-NEXT:    vmovdqa (%rax), %xmm02267; AVX512-NEXT:    vmovdqa (%r10), %xmm22268; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]2269; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm212270; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm222271; AVX512-NEXT:    vmovdqa (%r9), %xmm02272; AVX512-NEXT:    vmovdqa (%r8), %xmm22273; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]2274; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm232275; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm242276; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]2277; AVX512-NEXT:    vpermt2d %zmm1, %zmm16, %zmm142278; AVX512-NEXT:    vmovdqa (%rcx), %xmm62279; AVX512-NEXT:    vmovdqa (%rdx), %xmm72280; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]2281; AVX512-NEXT:    vmovdqa (%rsi), %xmm82282; AVX512-NEXT:    vmovdqa (%rdi), %xmm102283; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm10[4],xmm8[4],xmm10[5],xmm8[5],xmm10[6],xmm8[6],xmm10[7],xmm8[7]2284; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]2285; AVX512-NEXT:    vpermt2d %zmm15, %zmm17, %zmm52286; AVX512-NEXT:    movb $-86, %al2287; AVX512-NEXT:    kmovw %eax, %k12288; AVX512-NEXT:    vmovdqa64 %zmm14, %zmm5 {%k1}2289; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm12[0],ymm13[0],ymm12[1],ymm13[1],ymm12[2],ymm13[2],ymm12[3],ymm13[3],ymm12[8],ymm13[8],ymm12[9],ymm13[9],ymm12[10],ymm13[10],ymm12[11],ymm13[11]2290; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm9[0],ymm11[0],ymm9[1],ymm11[1],ymm9[2],ymm11[2],ymm9[3],ymm11[3],ymm9[8],ymm11[8],ymm9[9],ymm11[9],ymm9[10],ymm11[10],ymm9[11],ymm11[11]2291; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]2292; AVX512-NEXT:    vpermt2d %zmm14, %zmm18, %zmm152293; AVX512-NEXT:    vmovdqa (%rsi), %ymm42294; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]2295; AVX512-NEXT:    vmovdqa (%rdx), %ymm22296; AVX512-NEXT:    vmovdqa (%rcx), %ymm02297; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]2298; AVX512-NEXT:    vmovdqa64 %ymm20, %ymm12299; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm1[0],ymm4[0],ymm1[1],ymm4[1],ymm1[2],ymm4[2],ymm1[3],ymm4[3],ymm1[8],ymm4[8],ymm1[9],ymm4[9],ymm1[10],ymm4[10],ymm1[11],ymm4[11]2300; AVX512-NEXT:    vpermt2d %zmm3, %zmm19, %zmm142301; AVX512-NEXT:    vmovdqa64 %zmm15, %zmm14 {%k1}2302; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm12[4],ymm13[4],ymm12[5],ymm13[5],ymm12[6],ymm13[6],ymm12[7],ymm13[7],ymm12[12],ymm13[12],ymm12[13],ymm13[13],ymm12[14],ymm13[14],ymm12[15],ymm13[15]2303; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm9 = ymm9[4],ymm11[4],ymm9[5],ymm11[5],ymm9[6],ymm11[6],ymm9[7],ymm11[7],ymm9[12],ymm11[12],ymm9[13],ymm11[13],ymm9[14],ymm11[14],ymm9[15],ymm11[15]2304; AVX512-NEXT:    vpermt2d %zmm3, %zmm18, %zmm92305; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]2306; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm4[4],ymm1[5],ymm4[5],ymm1[6],ymm4[6],ymm1[7],ymm4[7],ymm1[12],ymm4[12],ymm1[13],ymm4[13],ymm1[14],ymm4[14],ymm1[15],ymm4[15]2307; AVX512-NEXT:    vpermt2d %zmm0, %zmm19, %zmm12308; AVX512-NEXT:    vmovdqa64 %zmm9, %zmm1 {%k1}2309; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm02310; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm22311; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]2312; AVX512-NEXT:    vmovdqa64 %xmm23, %xmm22313; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm32314; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]2315; AVX512-NEXT:    vpermt2d %zmm0, %zmm16, %zmm22316; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]2317; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]2318; AVX512-NEXT:    vpermt2d %zmm0, %zmm17, %zmm42319; AVX512-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}2320; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax2321; AVX512-NEXT:    vmovdqa64 %zmm4, (%rax)2322; AVX512-NEXT:    vmovdqa64 %zmm1, 192(%rax)2323; AVX512-NEXT:    vmovdqa64 %zmm14, 128(%rax)2324; AVX512-NEXT:    vmovdqa64 %zmm5, 64(%rax)2325; AVX512-NEXT:    vzeroupper2326; AVX512-NEXT:    retq2327;2328; AVX512-FCP-LABEL: store_i16_stride8_vf16:2329; AVX512-FCP:       # %bb.0:2330; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax2331; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r102332; AVX512-FCP-NEXT:    vmovdqa64 (%rdi), %ymm202333; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm92334; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm112335; AVX512-FCP-NEXT:    vmovdqa (%r10), %ymm122336; AVX512-FCP-NEXT:    vmovdqa (%rax), %ymm132337; AVX512-FCP-NEXT:    vmovdqa (%rax), %xmm02338; AVX512-FCP-NEXT:    vmovdqa (%r10), %xmm22339; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]2340; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm212341; AVX512-FCP-NEXT:    vmovdqa64 %xmm0, %xmm222342; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm02343; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm22344; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]2345; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm232346; AVX512-FCP-NEXT:    vmovdqa64 %xmm0, %xmm242347; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]2348; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm16, %zmm142349; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm62350; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm72351; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]2352; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm82353; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm102354; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm10[4],xmm8[4],xmm10[5],xmm8[5],xmm10[6],xmm8[6],xmm10[7],xmm8[7]2355; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]2356; AVX512-FCP-NEXT:    vpermt2d %zmm15, %zmm17, %zmm52357; AVX512-FCP-NEXT:    movb $-86, %al2358; AVX512-FCP-NEXT:    kmovw %eax, %k12359; AVX512-FCP-NEXT:    vmovdqa64 %zmm14, %zmm5 {%k1}2360; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm12[0],ymm13[0],ymm12[1],ymm13[1],ymm12[2],ymm13[2],ymm12[3],ymm13[3],ymm12[8],ymm13[8],ymm12[9],ymm13[9],ymm12[10],ymm13[10],ymm12[11],ymm13[11]2361; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm9[0],ymm11[0],ymm9[1],ymm11[1],ymm9[2],ymm11[2],ymm9[3],ymm11[3],ymm9[8],ymm11[8],ymm9[9],ymm11[9],ymm9[10],ymm11[10],ymm9[11],ymm11[11]2362; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]2363; AVX512-FCP-NEXT:    vpermt2d %zmm14, %zmm18, %zmm152364; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm42365; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]2366; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm22367; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm02368; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]2369; AVX512-FCP-NEXT:    vmovdqa64 %ymm20, %ymm12370; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm1[0],ymm4[0],ymm1[1],ymm4[1],ymm1[2],ymm4[2],ymm1[3],ymm4[3],ymm1[8],ymm4[8],ymm1[9],ymm4[9],ymm1[10],ymm4[10],ymm1[11],ymm4[11]2371; AVX512-FCP-NEXT:    vpermt2d %zmm3, %zmm19, %zmm142372; AVX512-FCP-NEXT:    vmovdqa64 %zmm15, %zmm14 {%k1}2373; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm12[4],ymm13[4],ymm12[5],ymm13[5],ymm12[6],ymm13[6],ymm12[7],ymm13[7],ymm12[12],ymm13[12],ymm12[13],ymm13[13],ymm12[14],ymm13[14],ymm12[15],ymm13[15]2374; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm9 = ymm9[4],ymm11[4],ymm9[5],ymm11[5],ymm9[6],ymm11[6],ymm9[7],ymm11[7],ymm9[12],ymm11[12],ymm9[13],ymm11[13],ymm9[14],ymm11[14],ymm9[15],ymm11[15]2375; AVX512-FCP-NEXT:    vpermt2d %zmm3, %zmm18, %zmm92376; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]2377; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm4[4],ymm1[5],ymm4[5],ymm1[6],ymm4[6],ymm1[7],ymm4[7],ymm1[12],ymm4[12],ymm1[13],ymm4[13],ymm1[14],ymm4[14],ymm1[15],ymm4[15]2378; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm19, %zmm12379; AVX512-FCP-NEXT:    vmovdqa64 %zmm9, %zmm1 {%k1}2380; AVX512-FCP-NEXT:    vmovdqa64 %xmm21, %xmm02381; AVX512-FCP-NEXT:    vmovdqa64 %xmm22, %xmm22382; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]2383; AVX512-FCP-NEXT:    vmovdqa64 %xmm23, %xmm22384; AVX512-FCP-NEXT:    vmovdqa64 %xmm24, %xmm32385; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]2386; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm16, %zmm22387; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]2388; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]2389; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm17, %zmm42390; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}2391; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax2392; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)2393; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 192(%rax)2394; AVX512-FCP-NEXT:    vmovdqa64 %zmm14, 128(%rax)2395; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, 64(%rax)2396; AVX512-FCP-NEXT:    vzeroupper2397; AVX512-FCP-NEXT:    retq2398;2399; AVX512DQ-LABEL: store_i16_stride8_vf16:2400; AVX512DQ:       # %bb.0:2401; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax2402; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %r102403; AVX512DQ-NEXT:    vmovdqa64 (%rdi), %ymm202404; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm92405; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm112406; AVX512DQ-NEXT:    vmovdqa (%r10), %ymm122407; AVX512DQ-NEXT:    vmovdqa (%rax), %ymm132408; AVX512DQ-NEXT:    vmovdqa (%rax), %xmm02409; AVX512DQ-NEXT:    vmovdqa (%r10), %xmm22410; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]2411; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm212412; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm222413; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm02414; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm22415; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]2416; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm232417; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm242418; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]2419; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm16, %zmm142420; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm62421; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm72422; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]2423; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm82424; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm102425; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm10[4],xmm8[4],xmm10[5],xmm8[5],xmm10[6],xmm8[6],xmm10[7],xmm8[7]2426; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]2427; AVX512DQ-NEXT:    vpermt2d %zmm15, %zmm17, %zmm52428; AVX512DQ-NEXT:    movb $-86, %al2429; AVX512DQ-NEXT:    kmovw %eax, %k12430; AVX512DQ-NEXT:    vmovdqa64 %zmm14, %zmm5 {%k1}2431; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm12[0],ymm13[0],ymm12[1],ymm13[1],ymm12[2],ymm13[2],ymm12[3],ymm13[3],ymm12[8],ymm13[8],ymm12[9],ymm13[9],ymm12[10],ymm13[10],ymm12[11],ymm13[11]2432; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm9[0],ymm11[0],ymm9[1],ymm11[1],ymm9[2],ymm11[2],ymm9[3],ymm11[3],ymm9[8],ymm11[8],ymm9[9],ymm11[9],ymm9[10],ymm11[10],ymm9[11],ymm11[11]2433; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]2434; AVX512DQ-NEXT:    vpermt2d %zmm14, %zmm18, %zmm152435; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm42436; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]2437; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm22438; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm02439; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]2440; AVX512DQ-NEXT:    vmovdqa64 %ymm20, %ymm12441; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm1[0],ymm4[0],ymm1[1],ymm4[1],ymm1[2],ymm4[2],ymm1[3],ymm4[3],ymm1[8],ymm4[8],ymm1[9],ymm4[9],ymm1[10],ymm4[10],ymm1[11],ymm4[11]2442; AVX512DQ-NEXT:    vpermt2d %zmm3, %zmm19, %zmm142443; AVX512DQ-NEXT:    vmovdqa64 %zmm15, %zmm14 {%k1}2444; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm12[4],ymm13[4],ymm12[5],ymm13[5],ymm12[6],ymm13[6],ymm12[7],ymm13[7],ymm12[12],ymm13[12],ymm12[13],ymm13[13],ymm12[14],ymm13[14],ymm12[15],ymm13[15]2445; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm9 = ymm9[4],ymm11[4],ymm9[5],ymm11[5],ymm9[6],ymm11[6],ymm9[7],ymm11[7],ymm9[12],ymm11[12],ymm9[13],ymm11[13],ymm9[14],ymm11[14],ymm9[15],ymm11[15]2446; AVX512DQ-NEXT:    vpermt2d %zmm3, %zmm18, %zmm92447; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]2448; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm4[4],ymm1[5],ymm4[5],ymm1[6],ymm4[6],ymm1[7],ymm4[7],ymm1[12],ymm4[12],ymm1[13],ymm4[13],ymm1[14],ymm4[14],ymm1[15],ymm4[15]2449; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm19, %zmm12450; AVX512DQ-NEXT:    vmovdqa64 %zmm9, %zmm1 {%k1}2451; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm02452; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm22453; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]2454; AVX512DQ-NEXT:    vmovdqa64 %xmm23, %xmm22455; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm32456; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]2457; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm16, %zmm22458; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]2459; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]2460; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm17, %zmm42461; AVX512DQ-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}2462; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax2463; AVX512DQ-NEXT:    vmovdqa64 %zmm4, (%rax)2464; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 192(%rax)2465; AVX512DQ-NEXT:    vmovdqa64 %zmm14, 128(%rax)2466; AVX512DQ-NEXT:    vmovdqa64 %zmm5, 64(%rax)2467; AVX512DQ-NEXT:    vzeroupper2468; AVX512DQ-NEXT:    retq2469;2470; AVX512DQ-FCP-LABEL: store_i16_stride8_vf16:2471; AVX512DQ-FCP:       # %bb.0:2472; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax2473; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r102474; AVX512DQ-FCP-NEXT:    vmovdqa64 (%rdi), %ymm202475; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm92476; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %ymm112477; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %ymm122478; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %ymm132479; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %xmm02480; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %xmm22481; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]2482; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm212483; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm0, %xmm222484; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm02485; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm22486; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]2487; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm232488; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm0, %xmm242489; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]2490; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm16, %zmm142491; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm62492; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm72493; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]2494; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm82495; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm102496; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm10[4],xmm8[4],xmm10[5],xmm8[5],xmm10[6],xmm8[6],xmm10[7],xmm8[7]2497; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]2498; AVX512DQ-FCP-NEXT:    vpermt2d %zmm15, %zmm17, %zmm52499; AVX512DQ-FCP-NEXT:    movb $-86, %al2500; AVX512DQ-FCP-NEXT:    kmovw %eax, %k12501; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm14, %zmm5 {%k1}2502; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm12[0],ymm13[0],ymm12[1],ymm13[1],ymm12[2],ymm13[2],ymm12[3],ymm13[3],ymm12[8],ymm13[8],ymm12[9],ymm13[9],ymm12[10],ymm13[10],ymm12[11],ymm13[11]2503; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm9[0],ymm11[0],ymm9[1],ymm11[1],ymm9[2],ymm11[2],ymm9[3],ymm11[3],ymm9[8],ymm11[8],ymm9[9],ymm11[9],ymm9[10],ymm11[10],ymm9[11],ymm11[11]2504; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]2505; AVX512DQ-FCP-NEXT:    vpermt2d %zmm14, %zmm18, %zmm152506; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm42507; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]2508; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm22509; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm02510; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]2511; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm20, %ymm12512; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm1[0],ymm4[0],ymm1[1],ymm4[1],ymm1[2],ymm4[2],ymm1[3],ymm4[3],ymm1[8],ymm4[8],ymm1[9],ymm4[9],ymm1[10],ymm4[10],ymm1[11],ymm4[11]2513; AVX512DQ-FCP-NEXT:    vpermt2d %zmm3, %zmm19, %zmm142514; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm15, %zmm14 {%k1}2515; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm12[4],ymm13[4],ymm12[5],ymm13[5],ymm12[6],ymm13[6],ymm12[7],ymm13[7],ymm12[12],ymm13[12],ymm12[13],ymm13[13],ymm12[14],ymm13[14],ymm12[15],ymm13[15]2516; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm9 = ymm9[4],ymm11[4],ymm9[5],ymm11[5],ymm9[6],ymm11[6],ymm9[7],ymm11[7],ymm9[12],ymm11[12],ymm9[13],ymm11[13],ymm9[14],ymm11[14],ymm9[15],ymm11[15]2517; AVX512DQ-FCP-NEXT:    vpermt2d %zmm3, %zmm18, %zmm92518; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]2519; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm4[4],ymm1[5],ymm4[5],ymm1[6],ymm4[6],ymm1[7],ymm4[7],ymm1[12],ymm4[12],ymm1[13],ymm4[13],ymm1[14],ymm4[14],ymm1[15],ymm4[15]2520; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm19, %zmm12521; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm9, %zmm1 {%k1}2522; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm21, %xmm02523; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm22, %xmm22524; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]2525; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm23, %xmm22526; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm24, %xmm32527; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]2528; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm16, %zmm22529; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]2530; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]2531; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm17, %zmm42532; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}2533; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax2534; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)2535; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 192(%rax)2536; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm14, 128(%rax)2537; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, 64(%rax)2538; AVX512DQ-FCP-NEXT:    vzeroupper2539; AVX512DQ-FCP-NEXT:    retq2540;2541; AVX512BW-LABEL: store_i16_stride8_vf16:2542; AVX512BW:       # %bb.0:2543; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax2544; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r102545; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r112546; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm02547; AVX512BW-NEXT:    vmovdqa (%rdx), %ymm12548; AVX512BW-NEXT:    vmovdqa (%r8), %ymm22549; AVX512BW-NEXT:    vmovdqa (%r11), %ymm32550; AVX512BW-NEXT:    vinserti64x4 $1, (%rsi), %zmm0, %zmm02551; AVX512BW-NEXT:    vinserti64x4 $1, (%rcx), %zmm1, %zmm12552; AVX512BW-NEXT:    vinserti64x4 $1, (%r9), %zmm2, %zmm22553; AVX512BW-NEXT:    vinserti64x4 $1, (%r10), %zmm3, %zmm32554; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,0,16,32,48,0,0,0,0,1,17,33,49,0,0,0,0,2,18,34,50,0,0,0,0,3,19,35,51]2555; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42556; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [0,16,32,48,0,0,0,0,1,17,33,49,0,0,0,0,2,18,34,50,0,0,0,0,3,19,35,51,0,0,0,0]2557; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm52558; AVX512BW-NEXT:    movb $-86, %cl2559; AVX512BW-NEXT:    kmovd %ecx, %k12560; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm5 {%k1}2561; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,4,20,36,52,0,0,0,0,5,21,37,53,0,0,0,0,6,22,38,54,0,0,0,0,7,23,39,55]2562; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42563; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [4,20,36,52,0,0,0,0,5,21,37,53,0,0,0,0,6,22,38,54,0,0,0,0,7,23,39,55,0,0,0,0]2564; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm62565; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm6 {%k1}2566; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,8,24,40,56,0,0,0,0,9,25,41,57,0,0,0,0,10,26,42,58,0,0,0,0,11,27,43,59]2567; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42568; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [8,24,40,56,0,0,0,0,9,25,41,57,0,0,0,0,10,26,42,58,0,0,0,0,11,27,43,59,0,0,0,0]2569; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm72570; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm7 {%k1}2571; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,12,28,44,60,0,0,0,0,13,29,45,61,0,0,0,0,14,30,46,62,0,0,0,0,15,31,47,63]2572; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42573; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [12,28,44,60,0,0,0,0,13,29,45,61,0,0,0,0,14,30,46,62,0,0,0,0,15,31,47,63,0,0,0,0]2574; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm22575; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm2 {%k1}2576; AVX512BW-NEXT:    vmovdqa64 %zmm2, 192(%rax)2577; AVX512BW-NEXT:    vmovdqa64 %zmm7, 128(%rax)2578; AVX512BW-NEXT:    vmovdqa64 %zmm6, 64(%rax)2579; AVX512BW-NEXT:    vmovdqa64 %zmm5, (%rax)2580; AVX512BW-NEXT:    vzeroupper2581; AVX512BW-NEXT:    retq2582;2583; AVX512BW-FCP-LABEL: store_i16_stride8_vf16:2584; AVX512BW-FCP:       # %bb.0:2585; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax2586; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r102587; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r112588; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %ymm02589; AVX512BW-FCP-NEXT:    vmovdqa (%rdx), %ymm12590; AVX512BW-FCP-NEXT:    vmovdqa (%r8), %ymm22591; AVX512BW-FCP-NEXT:    vmovdqa (%r11), %ymm32592; AVX512BW-FCP-NEXT:    vinserti64x4 $1, (%rsi), %zmm0, %zmm02593; AVX512BW-FCP-NEXT:    vinserti64x4 $1, (%rcx), %zmm1, %zmm12594; AVX512BW-FCP-NEXT:    vinserti64x4 $1, (%r9), %zmm2, %zmm22595; AVX512BW-FCP-NEXT:    vinserti64x4 $1, (%r10), %zmm3, %zmm32596; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,0,16,32,48,0,0,0,0,1,17,33,49,0,0,0,0,2,18,34,50,0,0,0,0,3,19,35,51]2597; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42598; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [0,16,32,48,0,0,0,0,1,17,33,49,0,0,0,0,2,18,34,50,0,0,0,0,3,19,35,51,0,0,0,0]2599; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm52600; AVX512BW-FCP-NEXT:    movb $-86, %cl2601; AVX512BW-FCP-NEXT:    kmovd %ecx, %k12602; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm5 {%k1}2603; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,4,20,36,52,0,0,0,0,5,21,37,53,0,0,0,0,6,22,38,54,0,0,0,0,7,23,39,55]2604; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42605; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [4,20,36,52,0,0,0,0,5,21,37,53,0,0,0,0,6,22,38,54,0,0,0,0,7,23,39,55,0,0,0,0]2606; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm62607; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm6 {%k1}2608; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,8,24,40,56,0,0,0,0,9,25,41,57,0,0,0,0,10,26,42,58,0,0,0,0,11,27,43,59]2609; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42610; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [8,24,40,56,0,0,0,0,9,25,41,57,0,0,0,0,10,26,42,58,0,0,0,0,11,27,43,59,0,0,0,0]2611; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm72612; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm7 {%k1}2613; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,12,28,44,60,0,0,0,0,13,29,45,61,0,0,0,0,14,30,46,62,0,0,0,0,15,31,47,63]2614; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42615; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [12,28,44,60,0,0,0,0,13,29,45,61,0,0,0,0,14,30,46,62,0,0,0,0,15,31,47,63,0,0,0,0]2616; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm22617; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm2 {%k1}2618; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm2, 192(%rax)2619; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm7, 128(%rax)2620; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm6, 64(%rax)2621; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm5, (%rax)2622; AVX512BW-FCP-NEXT:    vzeroupper2623; AVX512BW-FCP-NEXT:    retq2624;2625; AVX512DQ-BW-LABEL: store_i16_stride8_vf16:2626; AVX512DQ-BW:       # %bb.0:2627; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax2628; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %r102629; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %r112630; AVX512DQ-BW-NEXT:    vmovdqa (%rdi), %ymm02631; AVX512DQ-BW-NEXT:    vmovdqa (%rdx), %ymm12632; AVX512DQ-BW-NEXT:    vmovdqa (%r8), %ymm22633; AVX512DQ-BW-NEXT:    vmovdqa (%r11), %ymm32634; AVX512DQ-BW-NEXT:    vinserti64x4 $1, (%rsi), %zmm0, %zmm02635; AVX512DQ-BW-NEXT:    vinserti64x4 $1, (%rcx), %zmm1, %zmm12636; AVX512DQ-BW-NEXT:    vinserti64x4 $1, (%r9), %zmm2, %zmm22637; AVX512DQ-BW-NEXT:    vinserti64x4 $1, (%r10), %zmm3, %zmm32638; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,0,16,32,48,0,0,0,0,1,17,33,49,0,0,0,0,2,18,34,50,0,0,0,0,3,19,35,51]2639; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42640; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [0,16,32,48,0,0,0,0,1,17,33,49,0,0,0,0,2,18,34,50,0,0,0,0,3,19,35,51,0,0,0,0]2641; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm52642; AVX512DQ-BW-NEXT:    movb $-86, %cl2643; AVX512DQ-BW-NEXT:    kmovd %ecx, %k12644; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, %zmm5 {%k1}2645; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,4,20,36,52,0,0,0,0,5,21,37,53,0,0,0,0,6,22,38,54,0,0,0,0,7,23,39,55]2646; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42647; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [4,20,36,52,0,0,0,0,5,21,37,53,0,0,0,0,6,22,38,54,0,0,0,0,7,23,39,55,0,0,0,0]2648; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm62649; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, %zmm6 {%k1}2650; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,8,24,40,56,0,0,0,0,9,25,41,57,0,0,0,0,10,26,42,58,0,0,0,0,11,27,43,59]2651; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42652; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [8,24,40,56,0,0,0,0,9,25,41,57,0,0,0,0,10,26,42,58,0,0,0,0,11,27,43,59,0,0,0,0]2653; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm72654; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, %zmm7 {%k1}2655; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,12,28,44,60,0,0,0,0,13,29,45,61,0,0,0,0,14,30,46,62,0,0,0,0,15,31,47,63]2656; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42657; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [12,28,44,60,0,0,0,0,13,29,45,61,0,0,0,0,14,30,46,62,0,0,0,0,15,31,47,63,0,0,0,0]2658; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm22659; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, %zmm2 {%k1}2660; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm2, 192(%rax)2661; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm7, 128(%rax)2662; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm6, 64(%rax)2663; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm5, (%rax)2664; AVX512DQ-BW-NEXT:    vzeroupper2665; AVX512DQ-BW-NEXT:    retq2666;2667; AVX512DQ-BW-FCP-LABEL: store_i16_stride8_vf16:2668; AVX512DQ-BW-FCP:       # %bb.0:2669; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax2670; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r102671; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r112672; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %ymm02673; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdx), %ymm12674; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r8), %ymm22675; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%r11), %ymm32676; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, (%rsi), %zmm0, %zmm02677; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, (%rcx), %zmm1, %zmm12678; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, (%r9), %zmm2, %zmm22679; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, (%r10), %zmm3, %zmm32680; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,0,16,32,48,0,0,0,0,1,17,33,49,0,0,0,0,2,18,34,50,0,0,0,0,3,19,35,51]2681; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42682; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [0,16,32,48,0,0,0,0,1,17,33,49,0,0,0,0,2,18,34,50,0,0,0,0,3,19,35,51,0,0,0,0]2683; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm52684; AVX512DQ-BW-FCP-NEXT:    movb $-86, %cl2685; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k12686; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm5 {%k1}2687; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,4,20,36,52,0,0,0,0,5,21,37,53,0,0,0,0,6,22,38,54,0,0,0,0,7,23,39,55]2688; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42689; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [4,20,36,52,0,0,0,0,5,21,37,53,0,0,0,0,6,22,38,54,0,0,0,0,7,23,39,55,0,0,0,0]2690; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm62691; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm6 {%k1}2692; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,8,24,40,56,0,0,0,0,9,25,41,57,0,0,0,0,10,26,42,58,0,0,0,0,11,27,43,59]2693; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42694; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [8,24,40,56,0,0,0,0,9,25,41,57,0,0,0,0,10,26,42,58,0,0,0,0,11,27,43,59,0,0,0,0]2695; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm72696; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm7 {%k1}2697; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,12,28,44,60,0,0,0,0,13,29,45,61,0,0,0,0,14,30,46,62,0,0,0,0,15,31,47,63]2698; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm42699; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [12,28,44,60,0,0,0,0,13,29,45,61,0,0,0,0,14,30,46,62,0,0,0,0,15,31,47,63,0,0,0,0]2700; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm22701; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm2 {%k1}2702; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm2, 192(%rax)2703; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm7, 128(%rax)2704; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm6, 64(%rax)2705; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm5, (%rax)2706; AVX512DQ-BW-FCP-NEXT:    vzeroupper2707; AVX512DQ-BW-FCP-NEXT:    retq2708  %in.vec0 = load <16 x i16>, ptr %in.vecptr0, align 642709  %in.vec1 = load <16 x i16>, ptr %in.vecptr1, align 642710  %in.vec2 = load <16 x i16>, ptr %in.vecptr2, align 642711  %in.vec3 = load <16 x i16>, ptr %in.vecptr3, align 642712  %in.vec4 = load <16 x i16>, ptr %in.vecptr4, align 642713  %in.vec5 = load <16 x i16>, ptr %in.vecptr5, align 642714  %in.vec6 = load <16 x i16>, ptr %in.vecptr6, align 642715  %in.vec7 = load <16 x i16>, ptr %in.vecptr7, align 642716  %1 = shufflevector <16 x i16> %in.vec0, <16 x i16> %in.vec1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>2717  %2 = shufflevector <16 x i16> %in.vec2, <16 x i16> %in.vec3, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>2718  %3 = shufflevector <16 x i16> %in.vec4, <16 x i16> %in.vec5, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>2719  %4 = shufflevector <16 x i16> %in.vec6, <16 x i16> %in.vec7, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>2720  %5 = shufflevector <32 x i16> %1, <32 x i16> %2, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>2721  %6 = shufflevector <32 x i16> %3, <32 x i16> %4, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>2722  %7 = shufflevector <64 x i16> %5, <64 x i16> %6, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>2723  %interleaved.vec = shufflevector <128 x i16> %7, <128 x i16> poison, <128 x i32> <i32 0, i32 16, i32 32, i32 48, i32 64, i32 80, i32 96, i32 112, i32 1, i32 17, i32 33, i32 49, i32 65, i32 81, i32 97, i32 113, i32 2, i32 18, i32 34, i32 50, i32 66, i32 82, i32 98, i32 114, i32 3, i32 19, i32 35, i32 51, i32 67, i32 83, i32 99, i32 115, i32 4, i32 20, i32 36, i32 52, i32 68, i32 84, i32 100, i32 116, i32 5, i32 21, i32 37, i32 53, i32 69, i32 85, i32 101, i32 117, i32 6, i32 22, i32 38, i32 54, i32 70, i32 86, i32 102, i32 118, i32 7, i32 23, i32 39, i32 55, i32 71, i32 87, i32 103, i32 119, i32 8, i32 24, i32 40, i32 56, i32 72, i32 88, i32 104, i32 120, i32 9, i32 25, i32 41, i32 57, i32 73, i32 89, i32 105, i32 121, i32 10, i32 26, i32 42, i32 58, i32 74, i32 90, i32 106, i32 122, i32 11, i32 27, i32 43, i32 59, i32 75, i32 91, i32 107, i32 123, i32 12, i32 28, i32 44, i32 60, i32 76, i32 92, i32 108, i32 124, i32 13, i32 29, i32 45, i32 61, i32 77, i32 93, i32 109, i32 125, i32 14, i32 30, i32 46, i32 62, i32 78, i32 94, i32 110, i32 126, i32 15, i32 31, i32 47, i32 63, i32 79, i32 95, i32 111, i32 127>2724  store <128 x i16> %interleaved.vec, ptr %out.vec, align 642725  ret void2726}2727 2728define void @store_i16_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %in.vecptr4, ptr %in.vecptr5, ptr %in.vecptr6, ptr %in.vecptr7, ptr %out.vec) nounwind {2729; SSE-LABEL: store_i16_stride8_vf32:2730; SSE:       # %bb.0:2731; SSE-NEXT:    subq $264, %rsp # imm = 0x1082732; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %rax2733; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %r102734; SSE-NEXT:    movdqa (%rdi), %xmm32735; SSE-NEXT:    movdqa (%rsi), %xmm82736; SSE-NEXT:    movdqa (%rdx), %xmm42737; SSE-NEXT:    movdqa (%rcx), %xmm102738; SSE-NEXT:    movdqa (%r8), %xmm62739; SSE-NEXT:    movdqa (%r9), %xmm92740; SSE-NEXT:    movdqa (%r10), %xmm72741; SSE-NEXT:    movdqa (%rax), %xmm112742; SSE-NEXT:    movdqa %xmm4, %xmm12743; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1],xmm1[2],xmm10[2],xmm1[3],xmm10[3]2744; SSE-NEXT:    movdqa %xmm3, %xmm02745; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3]2746; SSE-NEXT:    movdqa %xmm0, %xmm22747; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]2748; SSE-NEXT:    movdqa %xmm7, %xmm122749; SSE-NEXT:    punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]2750; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,0,0]2751; SSE-NEXT:    movdqa %xmm6, %xmm52752; SSE-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3]2753; SSE-NEXT:    pshufd {{.*#+}} xmm14 = xmm5[0,0,0,0]2754; SSE-NEXT:    punpckhdq {{.*#+}} xmm14 = xmm14[2],xmm13[2],xmm14[3],xmm13[3]2755; SSE-NEXT:    movsd {{.*#+}} xmm14 = xmm2[0],xmm14[1]2756; SSE-NEXT:    movapd %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2757; SSE-NEXT:    movdqa %xmm5, %xmm22758; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm12[0],xmm2[1],xmm12[1]2759; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm1[1,1,1,1]2760; SSE-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[1,1,1,1]2761; SSE-NEXT:    punpckldq {{.*#+}} xmm14 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]2762; SSE-NEXT:    shufps {{.*#+}} xmm14 = xmm14[0,1],xmm2[2,3]2763; SSE-NEXT:    movaps %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2764; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm12[2,2,2,2]2765; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[2,2,2,2]2766; SSE-NEXT:    punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm2[2],xmm13[3],xmm2[3]2767; SSE-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[3,3,3,3]2768; SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]2769; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm0[0],xmm13[1]2770; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2771; SSE-NEXT:    movdqa 16(%r8), %xmm02772; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm12[2],xmm5[3],xmm12[3]2773; SSE-NEXT:    movdqa 16(%r9), %xmm22774; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]2775; SSE-NEXT:    punpckldq {{.*#+}} xmm14 = xmm14[0],xmm1[0],xmm14[1],xmm1[1]2776; SSE-NEXT:    movdqa 16(%r10), %xmm12777; SSE-NEXT:    shufps {{.*#+}} xmm14 = xmm14[0,1],xmm5[2,3]2778; SSE-NEXT:    movaps %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2779; SSE-NEXT:    movdqa 16(%rax), %xmm52780; SSE-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm10[4],xmm4[5],xmm10[5],xmm4[6],xmm10[6],xmm4[7],xmm10[7]2781; SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm8[4],xmm3[5],xmm8[5],xmm3[6],xmm8[6],xmm3[7],xmm8[7]2782; SSE-NEXT:    movdqa %xmm3, %xmm82783; SSE-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]2784; SSE-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm11[4],xmm7[5],xmm11[5],xmm7[6],xmm11[6],xmm7[7],xmm11[7]2785; SSE-NEXT:    pshufd {{.*#+}} xmm10 = xmm7[0,0,0,0]2786; SSE-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm9[4],xmm6[5],xmm9[5],xmm6[6],xmm9[6],xmm6[7],xmm9[7]2787; SSE-NEXT:    pshufd {{.*#+}} xmm9 = xmm6[0,0,0,0]2788; SSE-NEXT:    punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm10[2],xmm9[3],xmm10[3]2789; SSE-NEXT:    movsd {{.*#+}} xmm9 = xmm8[0],xmm9[1]2790; SSE-NEXT:    movapd %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2791; SSE-NEXT:    movdqa %xmm6, %xmm82792; SSE-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]2793; SSE-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[1,1,1,1]2794; SSE-NEXT:    pshufd {{.*#+}} xmm10 = xmm3[1,1,1,1]2795; SSE-NEXT:    punpckldq {{.*#+}} xmm10 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]2796; SSE-NEXT:    shufps {{.*#+}} xmm10 = xmm10[0,1],xmm8[2,3]2797; SSE-NEXT:    movaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2798; SSE-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[2,2,2,2]2799; SSE-NEXT:    pshufd {{.*#+}} xmm9 = xmm6[2,2,2,2]2800; SSE-NEXT:    punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm8[2],xmm9[3],xmm8[3]2801; SSE-NEXT:    pshufd {{.*#+}} xmm8 = xmm3[3,3,3,3]2802; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]2803; SSE-NEXT:    movsd {{.*#+}} xmm9 = xmm3[0],xmm9[1]2804; SSE-NEXT:    movapd %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2805; SSE-NEXT:    punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm7[2],xmm6[3],xmm7[3]2806; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[3,3,3,3]2807; SSE-NEXT:    punpckldq {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1]2808; SSE-NEXT:    shufps {{.*#+}} xmm8 = xmm8[0,1],xmm6[2,3]2809; SSE-NEXT:    movaps %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2810; SSE-NEXT:    movdqa %xmm1, %xmm72811; SSE-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]2812; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[0,0,0,0]2813; SSE-NEXT:    movdqa %xmm0, %xmm62814; SSE-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]2815; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[0,0,0,0]2816; SSE-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]2817; SSE-NEXT:    movdqa %xmm4, %xmm132818; SSE-NEXT:    movdqa 16(%rdx), %xmm32819; SSE-NEXT:    movdqa 16(%rcx), %xmm82820; SSE-NEXT:    movdqa %xmm3, %xmm102821; SSE-NEXT:    punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]2822; SSE-NEXT:    movdqa 16(%rdi), %xmm42823; SSE-NEXT:    movdqa 16(%rsi), %xmm92824; SSE-NEXT:    movdqa %xmm4, %xmm112825; SSE-NEXT:    punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]2826; SSE-NEXT:    movdqa %xmm11, %xmm122827; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm10[0],xmm12[1],xmm10[1]2828; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm12[0],xmm13[1]2829; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2830; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm10[1,1,1,1]2831; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm11[1,1,1,1]2832; SSE-NEXT:    punpckldq {{.*#+}} xmm13 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]2833; SSE-NEXT:    movdqa %xmm6, %xmm122834; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm7[0],xmm12[1],xmm7[1]2835; SSE-NEXT:    shufps {{.*#+}} xmm13 = xmm13[0,1],xmm12[2,3]2836; SSE-NEXT:    movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2837; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm7[2,2,2,2]2838; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[2,2,2,2]2839; SSE-NEXT:    punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm12[2],xmm13[3],xmm12[3]2840; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[3,3,3,3]2841; SSE-NEXT:    punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm10[2],xmm11[3],xmm10[3]2842; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm11[0],xmm13[1]2843; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2844; SSE-NEXT:    punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm7[2],xmm6[3],xmm7[3]2845; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm10[3,3,3,3]2846; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm7[0],xmm12[1],xmm7[1]2847; SSE-NEXT:    shufps {{.*#+}} xmm12 = xmm12[0,1],xmm6[2,3]2848; SSE-NEXT:    movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2849; SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm8[4],xmm3[5],xmm8[5],xmm3[6],xmm8[6],xmm3[7],xmm8[7]2850; SSE-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm9[4],xmm4[5],xmm9[5],xmm4[6],xmm9[6],xmm4[7],xmm9[7]2851; SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]2852; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]2853; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,0,0,0]2854; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,0,0]2855; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm2[2],xmm5[3],xmm2[3]2856; SSE-NEXT:    movdqa %xmm4, %xmm22857; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]2858; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm2[0],xmm5[1]2859; SSE-NEXT:    movapd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2860; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,1,1]2861; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]2862; SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]2863; SSE-NEXT:    movdqa %xmm0, %xmm22864; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]2865; SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm2[2,3]2866; SSE-NEXT:    movaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2867; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,2,2,2]2868; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[2,2,2,2]2869; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm2[2],xmm5[3],xmm2[3]2870; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[3,3,3,3]2871; SSE-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]2872; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm4[0],xmm5[1]2873; SSE-NEXT:    movapd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2874; SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]2875; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[3,3,3,3]2876; SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]2877; SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,1],xmm0[2,3]2878; SSE-NEXT:    movaps %xmm2, (%rsp) # 16-byte Spill2879; SSE-NEXT:    movdqa 32(%r10), %xmm02880; SSE-NEXT:    movdqa 32(%rax), %xmm42881; SSE-NEXT:    movdqa %xmm0, %xmm62882; SSE-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]2883; SSE-NEXT:    movdqa 32(%r8), %xmm12884; SSE-NEXT:    movdqa 32(%r9), %xmm52885; SSE-NEXT:    movdqa %xmm1, %xmm72886; SSE-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]2887; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[0,0,0,0]2888; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[0,0,0,0]2889; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]2890; SSE-NEXT:    movdqa %xmm3, %xmm132891; SSE-NEXT:    movdqa 32(%rdx), %xmm22892; SSE-NEXT:    movdqa 32(%rcx), %xmm82893; SSE-NEXT:    movdqa %xmm2, %xmm102894; SSE-NEXT:    punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]2895; SSE-NEXT:    movdqa 32(%rdi), %xmm32896; SSE-NEXT:    movdqa 32(%rsi), %xmm92897; SSE-NEXT:    movdqa %xmm3, %xmm112898; SSE-NEXT:    punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]2899; SSE-NEXT:    movdqa %xmm11, %xmm122900; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm10[0],xmm12[1],xmm10[1]2901; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm12[0],xmm13[1]2902; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2903; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm10[1,1,1,1]2904; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm11[1,1,1,1]2905; SSE-NEXT:    punpckldq {{.*#+}} xmm13 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]2906; SSE-NEXT:    movdqa %xmm7, %xmm122907; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm6[0],xmm12[1],xmm6[1]2908; SSE-NEXT:    shufps {{.*#+}} xmm13 = xmm13[0,1],xmm12[2,3]2909; SSE-NEXT:    movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2910; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[2,2,2,2]2911; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[2,2,2,2]2912; SSE-NEXT:    punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm12[2],xmm13[3],xmm12[3]2913; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[3,3,3,3]2914; SSE-NEXT:    punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm10[2],xmm11[3],xmm10[3]2915; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm11[0],xmm13[1]2916; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2917; SSE-NEXT:    punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm6[2],xmm7[3],xmm6[3]2918; SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm10[3,3,3,3]2919; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm6[0],xmm12[1],xmm6[1]2920; SSE-NEXT:    shufps {{.*#+}} xmm12 = xmm12[0,1],xmm7[2,3]2921; SSE-NEXT:    movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2922; SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]2923; SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm9[4],xmm3[5],xmm9[5],xmm3[6],xmm9[6],xmm3[7],xmm9[7]2924; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]2925; SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]2926; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,0,0]2927; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,0,0]2928; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]2929; SSE-NEXT:    movdqa %xmm3, %xmm42930; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]2931; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm4[0],xmm5[1]2932; SSE-NEXT:    movapd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2933; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]2934; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]2935; SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]2936; SSE-NEXT:    movdqa %xmm1, %xmm42937; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]2938; SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm4[2,3]2939; SSE-NEXT:    movaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2940; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,2,2,2]2941; SSE-NEXT:    pshufd {{.*#+}} xmm15 = xmm1[2,2,2,2]2942; SSE-NEXT:    punpckhdq {{.*#+}} xmm15 = xmm15[2],xmm4[2],xmm15[3],xmm4[3]2943; SSE-NEXT:    pshufd {{.*#+}} xmm14 = xmm3[3,3,3,3]2944; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]2945; SSE-NEXT:    movsd {{.*#+}} xmm15 = xmm3[0],xmm15[1]2946; SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]2947; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]2948; SSE-NEXT:    punpckldq {{.*#+}} xmm14 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]2949; SSE-NEXT:    shufps {{.*#+}} xmm14 = xmm14[0,1],xmm1[2,3]2950; SSE-NEXT:    movdqa 48(%r10), %xmm92951; SSE-NEXT:    movdqa 48(%rax), %xmm02952; SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2953; SSE-NEXT:    movdqa %xmm9, %xmm52954; SSE-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]2955; SSE-NEXT:    movdqa 48(%r8), %xmm42956; SSE-NEXT:    movdqa 48(%r9), %xmm02957; SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill2958; SSE-NEXT:    movdqa %xmm4, %xmm12959; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]2960; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,0,0]2961; SSE-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[0,0,0,0]2962; SSE-NEXT:    punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm0[2],xmm11[3],xmm0[3]2963; SSE-NEXT:    movdqa 48(%rdx), %xmm62964; SSE-NEXT:    movdqa 48(%rcx), %xmm132965; SSE-NEXT:    movdqa %xmm6, %xmm32966; SSE-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm13[0],xmm3[1],xmm13[1],xmm3[2],xmm13[2],xmm3[3],xmm13[3]2967; SSE-NEXT:    movdqa 48(%rdi), %xmm22968; SSE-NEXT:    movdqa 48(%rsi), %xmm122969; SSE-NEXT:    movdqa %xmm2, %xmm02970; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm12[0],xmm0[1],xmm12[1],xmm0[2],xmm12[2],xmm0[3],xmm12[3]2971; SSE-NEXT:    movdqa %xmm0, %xmm72972; SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1]2973; SSE-NEXT:    movsd {{.*#+}} xmm11 = xmm7[0],xmm11[1]2974; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,1,1]2975; SSE-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[1,1,1,1]2976; SSE-NEXT:    punpckldq {{.*#+}} xmm10 = xmm10[0],xmm7[0],xmm10[1],xmm7[1]2977; SSE-NEXT:    movdqa %xmm1, %xmm72978; SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]2979; SSE-NEXT:    shufps {{.*#+}} xmm10 = xmm10[0,1],xmm7[2,3]2980; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[2,2,2,2]2981; SSE-NEXT:    pshufd {{.*#+}} xmm8 = xmm1[2,2,2,2]2982; SSE-NEXT:    punpckhdq {{.*#+}} xmm8 = xmm8[2],xmm7[2],xmm8[3],xmm7[3]2983; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[3,3,3,3]2984; SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]2985; SSE-NEXT:    movsd {{.*#+}} xmm8 = xmm0[0],xmm8[1]2986; SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]2987; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[3,3,3,3]2988; SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1]2989; SSE-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,1],xmm1[2,3]2990; SSE-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm13[4],xmm6[5],xmm13[5],xmm6[6],xmm13[6],xmm6[7],xmm13[7]2991; SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm12[4],xmm2[5],xmm12[5],xmm2[6],xmm12[6],xmm2[7],xmm12[7]2992; SSE-NEXT:    punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload2993; SSE-NEXT:    # xmm9 = xmm9[4],mem[4],xmm9[5],mem[5],xmm9[6],mem[6],xmm9[7],mem[7]2994; SSE-NEXT:    punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload2995; SSE-NEXT:    # xmm4 = xmm4[4],mem[4],xmm4[5],mem[5],xmm4[6],mem[6],xmm4[7],mem[7]2996; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,0,0]2997; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[0,0,0,0]2998; SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]2999; SSE-NEXT:    movdqa %xmm2, %xmm03000; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]3001; SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]3002; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,1,1]3003; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,1,1]3004; SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]3005; SSE-NEXT:    movdqa %xmm4, %xmm03006; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1]3007; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm0[2,3]3008; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[2,2,2,2]3009; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[2,2,2,2]3010; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm0[2],xmm5[3],xmm0[3]3011; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]3012; SSE-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm6[2],xmm2[3],xmm6[3]3013; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm2[0],xmm5[1]3014; SSE-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm9[2],xmm4[3],xmm9[3]3015; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[3,3,3,3]3016; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]3017; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm4[2,3]3018; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %rax3019; SSE-NEXT:    movaps %xmm0, 496(%rax)3020; SSE-NEXT:    movapd %xmm5, 480(%rax)3021; SSE-NEXT:    movaps %xmm3, 464(%rax)3022; SSE-NEXT:    movapd %xmm1, 448(%rax)3023; SSE-NEXT:    movaps %xmm7, 432(%rax)3024; SSE-NEXT:    movapd %xmm8, 416(%rax)3025; SSE-NEXT:    movaps %xmm10, 400(%rax)3026; SSE-NEXT:    movapd %xmm11, 384(%rax)3027; SSE-NEXT:    movaps %xmm14, 368(%rax)3028; SSE-NEXT:    movapd %xmm15, 352(%rax)3029; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3030; SSE-NEXT:    movaps %xmm0, 336(%rax)3031; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3032; SSE-NEXT:    movaps %xmm0, 320(%rax)3033; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3034; SSE-NEXT:    movaps %xmm0, 304(%rax)3035; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3036; SSE-NEXT:    movaps %xmm0, 288(%rax)3037; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3038; SSE-NEXT:    movaps %xmm0, 272(%rax)3039; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3040; SSE-NEXT:    movaps %xmm0, 256(%rax)3041; SSE-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload3042; SSE-NEXT:    movaps %xmm0, 240(%rax)3043; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3044; SSE-NEXT:    movaps %xmm0, 224(%rax)3045; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3046; SSE-NEXT:    movaps %xmm0, 208(%rax)3047; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3048; SSE-NEXT:    movaps %xmm0, 192(%rax)3049; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3050; SSE-NEXT:    movaps %xmm0, 176(%rax)3051; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3052; SSE-NEXT:    movaps %xmm0, 160(%rax)3053; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3054; SSE-NEXT:    movaps %xmm0, 144(%rax)3055; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3056; SSE-NEXT:    movaps %xmm0, 128(%rax)3057; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3058; SSE-NEXT:    movaps %xmm0, 112(%rax)3059; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3060; SSE-NEXT:    movaps %xmm0, 96(%rax)3061; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3062; SSE-NEXT:    movaps %xmm0, 80(%rax)3063; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3064; SSE-NEXT:    movaps %xmm0, 64(%rax)3065; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3066; SSE-NEXT:    movaps %xmm0, 48(%rax)3067; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3068; SSE-NEXT:    movaps %xmm0, 32(%rax)3069; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3070; SSE-NEXT:    movaps %xmm0, 16(%rax)3071; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3072; SSE-NEXT:    movaps %xmm0, (%rax)3073; SSE-NEXT:    addq $264, %rsp # imm = 0x1083074; SSE-NEXT:    retq3075;3076; AVX-LABEL: store_i16_stride8_vf32:3077; AVX:       # %bb.0:3078; AVX-NEXT:    subq $296, %rsp # imm = 0x1283079; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax3080; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r103081; AVX-NEXT:    vmovdqa (%r9), %xmm03082; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3083; AVX-NEXT:    vmovdqa (%r8), %xmm13084; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3085; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3086; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm2[2,2,3,3]3087; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm03088; AVX-NEXT:    vmovdqa (%r10), %xmm13089; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3090; AVX-NEXT:    vmovdqa (%rax), %xmm33091; AVX-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3092; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]3093; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[2,2,2,2]3094; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm13095; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6],ymm1[7]3096; AVX-NEXT:    vmovdqa (%rcx), %xmm03097; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3098; AVX-NEXT:    vmovdqa (%rdx), %xmm13099; AVX-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3100; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3101; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm4[2,2,3,3]3102; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[2,3,2,3]3103; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm13104; AVX-NEXT:    vmovdqa (%rsi), %xmm03105; AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3106; AVX-NEXT:    vmovdqa (%rdi), %xmm63107; AVX-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3108; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]3109; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm6[2,3,2,3]3110; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[3,3,3,3]3111; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm73112; AVX-NEXT:    vmovdqa 48(%r9), %xmm03113; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0],ymm1[1],ymm7[2,3,4],ymm1[5],ymm7[6,7]3114; AVX-NEXT:    vmovdqa 48(%r10), %xmm13115; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm7[0,1],ymm5[2,3],ymm7[4,5],ymm5[6,7]3116; AVX-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3117; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[0,0,0,0]3118; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,1,0,1]3119; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm5, %ymm33120; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[0,1,0,1]3121; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero3122; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm53123; AVX-NEXT:    vmovdqa 48(%rax), %xmm23124; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1,2],ymm3[3],ymm5[4,5,6],ymm3[7]3125; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm6[1,1,1,1]3126; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm53127; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[0,0,1,1]3128; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm6, %ymm43129; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2,3,4],ymm4[5],ymm5[6,7]3130; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,3],ymm4[4,5],ymm3[6,7]3131; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3132; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]3133; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm8[0,0,0,0]3134; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm8[0,1,0,1]3135; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm43136; AVX-NEXT:    vmovdqa 48(%r8), %xmm33137; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]3138; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm9[0,1,0,1]3139; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm9[0],zero,xmm9[1],zero3140; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm53141; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm5[0,1,2],ymm4[3],ymm5[4,5,6],ymm4[7]3142; AVX-NEXT:    vmovdqa 48(%rsi), %xmm43143; AVX-NEXT:    vmovdqa 48(%rdi), %xmm53144; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]3145; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm11[1,1,1,1]3146; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm11, %ymm143147; AVX-NEXT:    vmovdqa 48(%rcx), %xmm63148; AVX-NEXT:    vmovdqa 48(%rdx), %xmm73149; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]3150; AVX-NEXT:    vpshufd {{.*#+}} xmm13 = xmm15[0,0,1,1]3151; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm13, %ymm133152; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm14[0],ymm13[1],ymm14[2,3,4],ymm13[5],ymm14[6,7]3153; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm13[0,1],ymm10[2,3],ymm13[4,5],ymm10[6,7]3154; AVX-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3155; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm9[2,2,3,3]3156; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm9, %ymm93157; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm8[2,2,2,2]3158; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm10, %ymm83159; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1,2],ymm8[3],ymm9[4,5,6],ymm8[7]3160; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm15[2,2,3,3]3161; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm15[2,3,2,3]3162; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm9, %ymm93163; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm11[2,3,2,3]3164; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]3165; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm10, %ymm103166; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0],ymm9[1],ymm10[2,3,4],ymm9[5],ymm10[6,7]3167; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1],ymm8[2,3],ymm9[4,5],ymm8[6,7]3168; AVX-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3169; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]3170; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[0,0,0,0]3171; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm1[0,1,0,1]3172; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm2, %ymm23173; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]3174; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[0,1,0,1]3175; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm0[0],zero,xmm0[1],zero3176; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm3, %ymm33177; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]3178; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]3179; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]3180; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]3181; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm53182; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[0,0,1,1]3183; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm6, %ymm63184; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm6[1],ymm5[2,3,4],ymm6[5],ymm5[6,7]3185; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm5[0,1],ymm2[2,3],ymm5[4,5],ymm2[6,7]3186; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3187; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]3188; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm03189; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,2,2,2]3190; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm13191; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6],ymm1[7]3192; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm4[2,2,3,3]3193; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]3194; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm03195; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]3196; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]3197; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm23198; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm0[1],ymm2[2,3,4],ymm0[5],ymm2[6,7]3199; AVX-NEXT:    vmovdqa 32(%r10), %xmm03200; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5],ymm1[6,7]3201; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3202; AVX-NEXT:    vmovdqa 32(%rax), %xmm13203; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]3204; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm8[0,0,0,0]3205; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm8[0,1,0,1]3206; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm43207; AVX-NEXT:    vmovdqa 32(%r9), %xmm23208; AVX-NEXT:    vmovdqa 32(%r8), %xmm33209; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]3210; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm9[0,1,0,1]3211; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm9[0],zero,xmm9[1],zero3212; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm53213; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm5[0,1,2],ymm4[3],ymm5[4,5,6],ymm4[7]3214; AVX-NEXT:    vmovdqa 32(%rsi), %xmm43215; AVX-NEXT:    vmovdqa 32(%rdi), %xmm53216; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]3217; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm11[1,1,1,1]3218; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm11, %ymm133219; AVX-NEXT:    vmovdqa 32(%rcx), %xmm63220; AVX-NEXT:    vmovdqa 32(%rdx), %xmm73221; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]3222; AVX-NEXT:    vpshufd {{.*#+}} xmm14 = xmm15[0,0,1,1]3223; AVX-NEXT:    vinsertf128 $1, %xmm15, %ymm14, %ymm143224; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0],ymm14[1],ymm13[2,3,4],ymm14[5],ymm13[6,7]3225; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm13[0,1],ymm10[2,3],ymm13[4,5],ymm10[6,7]3226; AVX-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3227; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm9[2,2,3,3]3228; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm9, %ymm93229; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm8[2,2,2,2]3230; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm10, %ymm83231; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1,2],ymm8[3],ymm9[4,5,6],ymm8[7]3232; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm15[2,2,3,3]3233; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm15[2,3,2,3]3234; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm9, %ymm93235; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm11[2,3,2,3]3236; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]3237; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm10, %ymm103238; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0],ymm9[1],ymm10[2,3,4],ymm9[5],ymm10[6,7]3239; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1],ymm8[2,3],ymm9[4,5],ymm8[6,7]3240; AVX-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3241; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3242; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]3243; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm0[0,1,0,1]3244; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm1, %ymm13245; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]3246; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[0,1,0,1]3247; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm2[0],zero,xmm2[1],zero3248; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm3, %ymm33249; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5,6],ymm1[7]3250; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]3251; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]3252; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]3253; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm53254; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[0,0,1,1]3255; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm6, %ymm63256; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm6[1],ymm5[2,3,4],ymm6[5],ymm5[6,7]3257; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm5[0,1],ymm1[2,3],ymm5[4,5],ymm1[6,7]3258; AVX-NEXT:    vmovups %ymm1, (%rsp) # 32-byte Spill3259; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]3260; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm13261; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,2,2,2]3262; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm03263; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6],ymm0[7]3264; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[2,2,3,3]3265; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]3266; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm13267; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]3268; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]3269; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm23270; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3,4],ymm1[5],ymm2[6,7]3271; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]3272; AVX-NEXT:    vmovdqa 16(%r10), %xmm123273; AVX-NEXT:    vmovdqa 16(%rax), %xmm113274; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]3275; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm8[0,0,0,0]3276; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm8[0,1,0,1]3277; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm03278; AVX-NEXT:    vmovdqa 16(%r9), %xmm103279; AVX-NEXT:    vmovdqa 16(%r8), %xmm73280; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm13 = xmm7[4],xmm10[4],xmm7[5],xmm10[5],xmm7[6],xmm10[6],xmm7[7],xmm10[7]3281; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm13[0,1,0,1]3282; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm13[0],zero,xmm13[1],zero3283; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm13284; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6],ymm0[7]3285; AVX-NEXT:    vmovdqa 16(%rsi), %xmm63286; AVX-NEXT:    vmovdqa 16(%rdi), %xmm53287; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]3288; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]3289; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm13290; AVX-NEXT:    vmovdqa 16(%rcx), %xmm43291; AVX-NEXT:    vmovdqa 16(%rdx), %xmm33292; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]3293; AVX-NEXT:    vpshufd {{.*#+}} xmm15 = xmm0[0,0,1,1]3294; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm15, %ymm153295; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm15[1],ymm1[2,3,4],ymm15[5],ymm1[6,7]3296; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm1[0,1],ymm9[2,3],ymm1[4,5],ymm9[6,7]3297; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm13[2,2,3,3]3298; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm13, %ymm13299; AVX-NEXT:    vpshufd {{.*#+}} xmm13 = xmm8[2,2,2,2]3300; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm13, %ymm83301; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm8[3],ymm1[4,5,6],ymm8[7]3302; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm0[2,2,3,3]3303; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]3304; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm8, %ymm03305; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm2[2,3,2,3]3306; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]3307; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm8, %ymm23308; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3,4],ymm0[5],ymm2[6,7]3309; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]3310; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]3311; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[0,0,0,0]3312; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[0,1,0,1]3313; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm03314; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm10[0],xmm7[1],xmm10[1],xmm7[2],xmm10[2],xmm7[3],xmm10[3]3315; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm2[0,1,0,1]3316; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm10 = xmm2[0],zero,xmm2[1],zero3317; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm7, %ymm73318; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm7[0,1,2],ymm0[3],ymm7[4,5,6],ymm0[7]3319; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]3320; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]3321; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm5[1,1,1,1]3322; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm43323; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[0,0,1,1]3324; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm6, %ymm63325; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0],ymm6[1],ymm4[2,3,4],ymm6[5],ymm4[6,7]3326; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1],ymm0[2,3],ymm4[4,5],ymm0[6,7]3327; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[2,2,3,3]3328; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm23329; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,2,2,2]3330; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm13331; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]3332; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,2,3,3]3333; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]3334; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm23335; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm5[2,3,2,3]3336; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm5[3,3,3,3]3337; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm33338; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3,4],ymm2[5],ymm3[6,7]3339; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5],ymm1[6,7]3340; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload3341; AVX-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload3342; AVX-NEXT:    # xmm2 = xmm2[4],mem[4],xmm2[5],mem[5],xmm2[6],mem[6],xmm2[7],mem[7]3343; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[0,0,0,0]3344; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[0,1,0,1]3345; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm33346; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload3347; AVX-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload3348; AVX-NEXT:    # xmm4 = xmm4[4],mem[4],xmm4[5],mem[5],xmm4[6],mem[6],xmm4[7],mem[7]3349; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[0,1,0,1]3350; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm4[0],zero,xmm4[1],zero3351; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm53352; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1,2],ymm3[3],ymm5[4,5,6],ymm3[7]3353; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload3354; AVX-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload3355; AVX-NEXT:    # xmm5 = xmm5[4],mem[4],xmm5[5],mem[5],xmm5[6],mem[6],xmm5[7],mem[7]3356; AVX-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload3357; AVX-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload3358; AVX-NEXT:    # xmm6 = xmm6[4],mem[4],xmm6[5],mem[5],xmm6[6],mem[6],xmm6[7],mem[7]3359; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm5[1,1,1,1]3360; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm5, %ymm73361; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm6[0,0,1,1]3362; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm10, %ymm103363; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0],ymm10[1],ymm7[2,3,4],ymm10[5],ymm7[6,7]3364; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm7[0,1],ymm3[2,3],ymm7[4,5],ymm3[6,7]3365; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm4[2,2,3,3]3366; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm4, %ymm43367; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm2[2,2,2,2]3368; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm7, %ymm23369; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3],ymm4[4,5,6],ymm2[7]3370; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm6[2,2,3,3]3371; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]3372; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm4, %ymm43373; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]3374; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[3,3,3,3]3375; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm53376; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2,3,4],ymm4[5],ymm5[6,7]3377; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1],ymm2[2,3],ymm4[4,5],ymm2[6,7]3378; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax3379; AVX-NEXT:    vmovaps %ymm2, 96(%rax)3380; AVX-NEXT:    vmovaps %ymm3, 64(%rax)3381; AVX-NEXT:    vmovaps %ymm1, 160(%rax)3382; AVX-NEXT:    vmovaps %ymm0, 128(%rax)3383; AVX-NEXT:    vmovaps %ymm8, 224(%rax)3384; AVX-NEXT:    vmovaps %ymm9, 192(%rax)3385; AVX-NEXT:    vmovaps %ymm14, 288(%rax)3386; AVX-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload3387; AVX-NEXT:    vmovaps %ymm0, 256(%rax)3388; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3389; AVX-NEXT:    vmovaps %ymm0, 352(%rax)3390; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3391; AVX-NEXT:    vmovaps %ymm0, 320(%rax)3392; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3393; AVX-NEXT:    vmovaps %ymm0, 416(%rax)3394; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3395; AVX-NEXT:    vmovaps %ymm0, 384(%rax)3396; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3397; AVX-NEXT:    vmovaps %ymm0, 480(%rax)3398; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3399; AVX-NEXT:    vmovaps %ymm0, 448(%rax)3400; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3401; AVX-NEXT:    vmovaps %ymm0, (%rax)3402; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3403; AVX-NEXT:    vmovaps %ymm0, 32(%rax)3404; AVX-NEXT:    addq $296, %rsp # imm = 0x1283405; AVX-NEXT:    vzeroupper3406; AVX-NEXT:    retq3407;3408; AVX2-LABEL: store_i16_stride8_vf32:3409; AVX2:       # %bb.0:3410; AVX2-NEXT:    subq $232, %rsp3411; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %rax3412; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r103413; AVX2-NEXT:    vmovdqa (%rax), %xmm03414; AVX2-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3415; AVX2-NEXT:    vmovdqa 32(%rax), %xmm103416; AVX2-NEXT:    vmovdqa (%r10), %xmm13417; AVX2-NEXT:    vmovdqa %xmm1, (%rsp) # 16-byte Spill3418; AVX2-NEXT:    vmovdqa 32(%r10), %xmm93419; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3420; AVX2-NEXT:    vmovdqa (%r9), %xmm03421; AVX2-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3422; AVX2-NEXT:    vmovdqa (%r8), %xmm23423; AVX2-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3424; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]3425; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]3426; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]3427; AVX2-NEXT:    vmovdqa (%rcx), %xmm43428; AVX2-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3429; AVX2-NEXT:    vmovdqa (%rdx), %xmm33430; AVX2-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3431; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]3432; AVX2-NEXT:    vmovdqa (%rsi), %xmm123433; AVX2-NEXT:    vmovdqa (%rdi), %xmm133434; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]3435; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm15 = xmm14[2],xmm3[2],xmm14[3],xmm3[3]3436; AVX2-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,1,1,3]3437; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm15[0,1],ymm0[2,3],ymm15[4,5],ymm0[6,7]3438; AVX2-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3439; AVX2-NEXT:    vmovdqa 32(%r9), %xmm83440; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]3441; AVX2-NEXT:    vmovdqa 32(%r8), %xmm153442; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]3443; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm14[0],xmm3[0],xmm14[1],xmm3[1]3444; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,1,3]3445; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm1[2,3],ymm2[4,5],ymm1[6,7]3446; AVX2-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3447; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm9[4],xmm10[4],xmm9[5],xmm10[5],xmm9[6],xmm10[6],xmm9[7],xmm10[7]3448; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm15[4],xmm8[4],xmm15[5],xmm8[5],xmm15[6],xmm8[6],xmm15[7],xmm8[7]3449; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm14[0],xmm3[0],xmm14[1],xmm3[1]3450; AVX2-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[0,0,2,1]3451; AVX2-NEXT:    vmovdqa 32(%rcx), %xmm63452; AVX2-NEXT:    vmovdqa 32(%rdx), %xmm53453; AVX2-NEXT:    vmovdqa 32(%rsi), %xmm43454; AVX2-NEXT:    vmovdqa 32(%rdi), %xmm23455; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]3456; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]3457; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm11 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]3458; AVX2-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,1,1,3]3459; AVX2-NEXT:    vpblendd {{.*#+}} ymm7 = ymm11[0,1],ymm7[2,3],ymm11[4,5],ymm7[6,7]3460; AVX2-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3461; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm14[2],xmm3[2],xmm14[3],xmm3[3]3462; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]3463; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm3[0,0,2,1]3464; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]3465; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]3466; AVX2-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3467; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]3468; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3]3469; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]3470; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]3471; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3472; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,0,2,1]3473; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]3474; AVX2-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,1,1,3]3475; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0,1],ymm4[2,3],ymm6[4,5],ymm4[6,7]3476; AVX2-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3477; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]3478; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm2[2],xmm5[2],xmm2[3],xmm5[3]3479; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm143480; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]3481; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]3482; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]3483; AVX2-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3484; AVX2-NEXT:    vmovdqa 32(%r8), %ymm153485; AVX2-NEXT:    vmovdqa (%rsp), %xmm0 # 16-byte Reload3486; AVX2-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload3487; AVX2-NEXT:    # xmm1 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]3488; AVX2-NEXT:    vmovdqa 32(%r9), %ymm43489; AVX2-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3490; AVX2-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload3491; AVX2-NEXT:    # xmm2 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]3492; AVX2-NEXT:    vmovdqa 32(%r10), %ymm63493; AVX2-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3494; AVX2-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm7 # 16-byte Folded Reload3495; AVX2-NEXT:    # xmm7 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]3496; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm8 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]3497; AVX2-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,2,1]3498; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7]3499; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm10 = xmm9[0],xmm7[0],xmm9[1],xmm7[1]3500; AVX2-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,1,1,3]3501; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1],ymm8[2,3],ymm10[4,5],ymm8[6,7]3502; AVX2-NEXT:    vmovdqu %ymm0, (%rsp) # 32-byte Spill3503; AVX2-NEXT:    vmovdqa 32(%rax), %ymm103504; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]3505; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]3506; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm9[2],xmm7[2],xmm9[3],xmm7[3]3507; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,1,3]3508; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm1[2,3],ymm2[4,5],ymm1[6,7]3509; AVX2-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3510; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm6[4],ymm10[4],ymm6[5],ymm10[5],ymm6[6],ymm10[6],ymm6[7],ymm10[7],ymm6[12],ymm10[12],ymm6[13],ymm10[13],ymm6[14],ymm10[14],ymm6[15],ymm10[15]3511; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm15[4],ymm4[4],ymm15[5],ymm4[5],ymm15[6],ymm4[6],ymm15[7],ymm4[7],ymm15[12],ymm4[12],ymm15[13],ymm4[13],ymm15[14],ymm4[14],ymm15[15],ymm4[15]3512; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm8 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5]3513; AVX2-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,2,3]3514; AVX2-NEXT:    vmovdqa 32(%rsi), %ymm113515; AVX2-NEXT:    vmovdqa 32(%rdx), %ymm133516; AVX2-NEXT:    vmovdqa 32(%rcx), %ymm93517; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm13[4],ymm9[4],ymm13[5],ymm9[5],ymm13[6],ymm9[6],ymm13[7],ymm9[7],ymm13[12],ymm9[12],ymm13[13],ymm9[13],ymm13[14],ymm9[14],ymm13[15],ymm9[15]3518; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm14[4],ymm11[4],ymm14[5],ymm11[5],ymm14[6],ymm11[6],ymm14[7],ymm11[7],ymm14[12],ymm11[12],ymm14[13],ymm11[13],ymm14[14],ymm11[14],ymm14[15],ymm11[15]3519; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm5 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]3520; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,1,3,3]3521; AVX2-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1],ymm8[2,3],ymm5[4,5],ymm8[6,7]3522; AVX2-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3523; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[6],ymm3[6],ymm2[7],ymm3[7]3524; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]3525; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm2[0,2,2,3]3526; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3]3527; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]3528; AVX2-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3529; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm6[0],ymm10[0],ymm6[1],ymm10[1],ymm6[2],ymm10[2],ymm6[3],ymm10[3],ymm6[8],ymm10[8],ymm6[9],ymm10[9],ymm6[10],ymm10[10],ymm6[11],ymm10[11]3530; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm15[0],ymm4[0],ymm15[1],ymm4[1],ymm15[2],ymm4[2],ymm15[3],ymm4[3],ymm15[8],ymm4[8],ymm15[9],ymm4[9],ymm15[10],ymm4[10],ymm15[11],ymm4[11]3531; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm13[0],ymm9[0],ymm13[1],ymm9[1],ymm13[2],ymm9[2],ymm13[3],ymm9[3],ymm13[8],ymm9[8],ymm13[9],ymm9[9],ymm13[10],ymm9[10],ymm13[11],ymm9[11]3532; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm4 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[4],ymm0[4],ymm2[5],ymm0[5]3533; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3]3534; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm14[0],ymm11[0],ymm14[1],ymm11[1],ymm14[2],ymm11[2],ymm14[3],ymm11[3],ymm14[8],ymm11[8],ymm14[9],ymm11[9],ymm14[10],ymm11[10],ymm14[11],ymm11[11]3535; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm6 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[4],ymm3[4],ymm5[5],ymm3[5]3536; AVX2-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,1,3,3]3537; AVX2-NEXT:    vpblendd {{.*#+}} ymm10 = ymm6[0,1],ymm4[2,3],ymm6[4,5],ymm4[6,7]3538; AVX2-NEXT:    vmovdqa (%r8), %ymm83539; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[6],ymm0[6],ymm2[7],ymm0[7]3540; AVX2-NEXT:    vmovdqa (%r9), %ymm73541; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm3 = ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[6],ymm3[6],ymm5[7],ymm3[7]3542; AVX2-NEXT:    vmovdqa (%r10), %ymm53543; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]3544; AVX2-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,1,3,3]3545; AVX2-NEXT:    vpblendd {{.*#+}} ymm11 = ymm3[0,1],ymm0[2,3],ymm3[4,5],ymm0[6,7]3546; AVX2-NEXT:    vmovdqa (%rax), %ymm43547; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm5[4],ymm4[4],ymm5[5],ymm4[5],ymm5[6],ymm4[6],ymm5[7],ymm4[7],ymm5[12],ymm4[12],ymm5[13],ymm4[13],ymm5[14],ymm4[14],ymm5[15],ymm4[15]3548; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm8[4],ymm7[4],ymm8[5],ymm7[5],ymm8[6],ymm7[6],ymm8[7],ymm7[7],ymm8[12],ymm7[12],ymm8[13],ymm7[13],ymm8[14],ymm7[14],ymm8[15],ymm7[15]3549; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm9 = ymm6[0],ymm3[0],ymm6[1],ymm3[1],ymm6[4],ymm3[4],ymm6[5],ymm3[5]3550; AVX2-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,2,3]3551; AVX2-NEXT:    vmovdqa (%rdi), %ymm133552; AVX2-NEXT:    vmovdqa (%rsi), %ymm143553; AVX2-NEXT:    vmovdqa (%rdx), %ymm153554; AVX2-NEXT:    vmovdqa (%rcx), %ymm23555; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm15[4],ymm2[4],ymm15[5],ymm2[5],ymm15[6],ymm2[6],ymm15[7],ymm2[7],ymm15[12],ymm2[12],ymm15[13],ymm2[13],ymm15[14],ymm2[14],ymm15[15],ymm2[15]3556; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm13[4],ymm14[4],ymm13[5],ymm14[5],ymm13[6],ymm14[6],ymm13[7],ymm14[7],ymm13[12],ymm14[12],ymm13[13],ymm14[13],ymm13[14],ymm14[14],ymm13[15],ymm14[15]3557; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm12 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]3558; AVX2-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,1,3,3]3559; AVX2-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0,1],ymm9[2,3],ymm12[4,5],ymm9[6,7]3560; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm3 = ymm6[2],ymm3[2],ymm6[3],ymm3[3],ymm6[6],ymm3[6],ymm6[7],ymm3[7]3561; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]3562; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm3[0,2,2,3]3563; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3]3564; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]3565; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[8],ymm4[8],ymm5[9],ymm4[9],ymm5[10],ymm4[10],ymm5[11],ymm4[11]3566; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm8[0],ymm7[0],ymm8[1],ymm7[1],ymm8[2],ymm7[2],ymm8[3],ymm7[3],ymm8[8],ymm7[8],ymm8[9],ymm7[9],ymm8[10],ymm7[10],ymm8[11],ymm7[11]3567; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm15[0],ymm2[0],ymm15[1],ymm2[1],ymm15[2],ymm2[2],ymm15[3],ymm2[3],ymm15[8],ymm2[8],ymm15[9],ymm2[9],ymm15[10],ymm2[10],ymm15[11],ymm2[11]3568; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm13[0],ymm14[0],ymm13[1],ymm14[1],ymm13[2],ymm14[2],ymm13[3],ymm14[3],ymm13[8],ymm14[8],ymm13[9],ymm14[9],ymm13[10],ymm14[10],ymm13[11],ymm14[11]3569; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm5 = ymm3[0],ymm1[0],ymm3[1],ymm1[1],ymm3[4],ymm1[4],ymm3[5],ymm1[5]3570; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,2,2,3]3571; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm6 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[4],ymm2[4],ymm4[5],ymm2[5]3572; AVX2-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,1,3,3]3573; AVX2-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0,1],ymm5[2,3],ymm6[4,5],ymm5[6,7]3574; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm1 = ymm3[2],ymm1[2],ymm3[3],ymm1[3],ymm3[6],ymm1[6],ymm3[7],ymm1[7]3575; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm4[2],ymm2[2],ymm4[3],ymm2[3],ymm4[6],ymm2[6],ymm4[7],ymm2[7]3576; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]3577; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,1,3,3]3578; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5],ymm1[6,7]3579; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %rax3580; AVX2-NEXT:    vmovdqa %ymm1, 160(%rax)3581; AVX2-NEXT:    vmovdqa %ymm5, 128(%rax)3582; AVX2-NEXT:    vmovdqa %ymm0, 224(%rax)3583; AVX2-NEXT:    vmovdqa %ymm9, 192(%rax)3584; AVX2-NEXT:    vmovdqa %ymm11, 416(%rax)3585; AVX2-NEXT:    vmovdqa %ymm10, 384(%rax)3586; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3587; AVX2-NEXT:    vmovaps %ymm0, 480(%rax)3588; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3589; AVX2-NEXT:    vmovaps %ymm0, 448(%rax)3590; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3591; AVX2-NEXT:    vmovaps %ymm0, 96(%rax)3592; AVX2-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload3593; AVX2-NEXT:    vmovaps %ymm0, 64(%rax)3594; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3595; AVX2-NEXT:    vmovaps %ymm0, 288(%rax)3596; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3597; AVX2-NEXT:    vmovaps %ymm0, 256(%rax)3598; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3599; AVX2-NEXT:    vmovaps %ymm0, 352(%rax)3600; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3601; AVX2-NEXT:    vmovaps %ymm0, 320(%rax)3602; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3603; AVX2-NEXT:    vmovaps %ymm0, (%rax)3604; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3605; AVX2-NEXT:    vmovaps %ymm0, 32(%rax)3606; AVX2-NEXT:    addq $232, %rsp3607; AVX2-NEXT:    vzeroupper3608; AVX2-NEXT:    retq3609;3610; AVX2-FP-LABEL: store_i16_stride8_vf32:3611; AVX2-FP:       # %bb.0:3612; AVX2-FP-NEXT:    subq $232, %rsp3613; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %rax3614; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %r103615; AVX2-FP-NEXT:    vmovdqa (%rax), %xmm03616; AVX2-FP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3617; AVX2-FP-NEXT:    vmovdqa 32(%rax), %xmm103618; AVX2-FP-NEXT:    vmovdqa (%r10), %xmm13619; AVX2-FP-NEXT:    vmovdqa %xmm1, (%rsp) # 16-byte Spill3620; AVX2-FP-NEXT:    vmovdqa 32(%r10), %xmm93621; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3622; AVX2-FP-NEXT:    vmovdqa (%r9), %xmm03623; AVX2-FP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3624; AVX2-FP-NEXT:    vmovdqa (%r8), %xmm23625; AVX2-FP-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3626; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]3627; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]3628; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]3629; AVX2-FP-NEXT:    vmovdqa (%rcx), %xmm43630; AVX2-FP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3631; AVX2-FP-NEXT:    vmovdqa (%rdx), %xmm33632; AVX2-FP-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3633; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]3634; AVX2-FP-NEXT:    vmovdqa (%rsi), %xmm123635; AVX2-FP-NEXT:    vmovdqa (%rdi), %xmm133636; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]3637; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm15 = xmm14[2],xmm3[2],xmm14[3],xmm3[3]3638; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,1,1,3]3639; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm15[0,1],ymm0[2,3],ymm15[4,5],ymm0[6,7]3640; AVX2-FP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3641; AVX2-FP-NEXT:    vmovdqa 32(%r9), %xmm83642; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]3643; AVX2-FP-NEXT:    vmovdqa 32(%r8), %xmm153644; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]3645; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm14[0],xmm3[0],xmm14[1],xmm3[1]3646; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,1,3]3647; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm1[2,3],ymm2[4,5],ymm1[6,7]3648; AVX2-FP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3649; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm9[4],xmm10[4],xmm9[5],xmm10[5],xmm9[6],xmm10[6],xmm9[7],xmm10[7]3650; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm15[4],xmm8[4],xmm15[5],xmm8[5],xmm15[6],xmm8[6],xmm15[7],xmm8[7]3651; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm14[0],xmm3[0],xmm14[1],xmm3[1]3652; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[0,0,2,1]3653; AVX2-FP-NEXT:    vmovdqa 32(%rcx), %xmm63654; AVX2-FP-NEXT:    vmovdqa 32(%rdx), %xmm53655; AVX2-FP-NEXT:    vmovdqa 32(%rsi), %xmm43656; AVX2-FP-NEXT:    vmovdqa 32(%rdi), %xmm23657; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]3658; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]3659; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm11 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]3660; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,1,1,3]3661; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm11[0,1],ymm7[2,3],ymm11[4,5],ymm7[6,7]3662; AVX2-FP-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3663; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm14[2],xmm3[2],xmm14[3],xmm3[3]3664; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]3665; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm3[0,0,2,1]3666; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]3667; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]3668; AVX2-FP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3669; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]3670; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3]3671; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]3672; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]3673; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]3674; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,0,2,1]3675; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]3676; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,1,1,3]3677; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0,1],ymm4[2,3],ymm6[4,5],ymm4[6,7]3678; AVX2-FP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3679; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]3680; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm2[2],xmm5[2],xmm2[3],xmm5[3]3681; AVX2-FP-NEXT:    vmovdqa 32(%rdi), %ymm143682; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]3683; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]3684; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]3685; AVX2-FP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3686; AVX2-FP-NEXT:    vmovdqa 32(%r8), %ymm153687; AVX2-FP-NEXT:    vmovdqa (%rsp), %xmm0 # 16-byte Reload3688; AVX2-FP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm1 # 16-byte Folded Reload3689; AVX2-FP-NEXT:    # xmm1 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]3690; AVX2-FP-NEXT:    vmovdqa 32(%r9), %ymm43691; AVX2-FP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3692; AVX2-FP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload3693; AVX2-FP-NEXT:    # xmm2 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]3694; AVX2-FP-NEXT:    vmovdqa 32(%r10), %ymm63695; AVX2-FP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3696; AVX2-FP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm7 # 16-byte Folded Reload3697; AVX2-FP-NEXT:    # xmm7 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]3698; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm8 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]3699; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,0,2,1]3700; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7]3701; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm10 = xmm9[0],xmm7[0],xmm9[1],xmm7[1]3702; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,1,1,3]3703; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1],ymm8[2,3],ymm10[4,5],ymm8[6,7]3704; AVX2-FP-NEXT:    vmovdqu %ymm0, (%rsp) # 32-byte Spill3705; AVX2-FP-NEXT:    vmovdqa 32(%rax), %ymm103706; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]3707; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,2,1]3708; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm9[2],xmm7[2],xmm9[3],xmm7[3]3709; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,1,1,3]3710; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm1[2,3],ymm2[4,5],ymm1[6,7]3711; AVX2-FP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3712; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm6[4],ymm10[4],ymm6[5],ymm10[5],ymm6[6],ymm10[6],ymm6[7],ymm10[7],ymm6[12],ymm10[12],ymm6[13],ymm10[13],ymm6[14],ymm10[14],ymm6[15],ymm10[15]3713; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm15[4],ymm4[4],ymm15[5],ymm4[5],ymm15[6],ymm4[6],ymm15[7],ymm4[7],ymm15[12],ymm4[12],ymm15[13],ymm4[13],ymm15[14],ymm4[14],ymm15[15],ymm4[15]3714; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm8 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[4],ymm3[4],ymm2[5],ymm3[5]3715; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,2,3]3716; AVX2-FP-NEXT:    vmovdqa 32(%rsi), %ymm113717; AVX2-FP-NEXT:    vmovdqa 32(%rdx), %ymm133718; AVX2-FP-NEXT:    vmovdqa 32(%rcx), %ymm93719; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm13[4],ymm9[4],ymm13[5],ymm9[5],ymm13[6],ymm9[6],ymm13[7],ymm9[7],ymm13[12],ymm9[12],ymm13[13],ymm9[13],ymm13[14],ymm9[14],ymm13[15],ymm9[15]3720; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm14[4],ymm11[4],ymm14[5],ymm11[5],ymm14[6],ymm11[6],ymm14[7],ymm11[7],ymm14[12],ymm11[12],ymm14[13],ymm11[13],ymm14[14],ymm11[14],ymm14[15],ymm11[15]3721; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm5 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]3722; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,1,3,3]3723; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1],ymm8[2,3],ymm5[4,5],ymm8[6,7]3724; AVX2-FP-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3725; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[6],ymm3[6],ymm2[7],ymm3[7]3726; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]3727; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm2[0,2,2,3]3728; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3]3729; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]3730; AVX2-FP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3731; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm6[0],ymm10[0],ymm6[1],ymm10[1],ymm6[2],ymm10[2],ymm6[3],ymm10[3],ymm6[8],ymm10[8],ymm6[9],ymm10[9],ymm6[10],ymm10[10],ymm6[11],ymm10[11]3732; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm15[0],ymm4[0],ymm15[1],ymm4[1],ymm15[2],ymm4[2],ymm15[3],ymm4[3],ymm15[8],ymm4[8],ymm15[9],ymm4[9],ymm15[10],ymm4[10],ymm15[11],ymm4[11]3733; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm13[0],ymm9[0],ymm13[1],ymm9[1],ymm13[2],ymm9[2],ymm13[3],ymm9[3],ymm13[8],ymm9[8],ymm13[9],ymm9[9],ymm13[10],ymm9[10],ymm13[11],ymm9[11]3734; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm4 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[4],ymm0[4],ymm2[5],ymm0[5]3735; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3]3736; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm14[0],ymm11[0],ymm14[1],ymm11[1],ymm14[2],ymm11[2],ymm14[3],ymm11[3],ymm14[8],ymm11[8],ymm14[9],ymm11[9],ymm14[10],ymm11[10],ymm14[11],ymm11[11]3737; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm6 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[4],ymm3[4],ymm5[5],ymm3[5]3738; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,1,3,3]3739; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm6[0,1],ymm4[2,3],ymm6[4,5],ymm4[6,7]3740; AVX2-FP-NEXT:    vmovdqa (%r8), %ymm83741; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[6],ymm0[6],ymm2[7],ymm0[7]3742; AVX2-FP-NEXT:    vmovdqa (%r9), %ymm73743; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm3 = ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[6],ymm3[6],ymm5[7],ymm3[7]3744; AVX2-FP-NEXT:    vmovdqa (%r10), %ymm53745; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]3746; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,1,3,3]3747; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm11 = ymm3[0,1],ymm0[2,3],ymm3[4,5],ymm0[6,7]3748; AVX2-FP-NEXT:    vmovdqa (%rax), %ymm43749; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm5[4],ymm4[4],ymm5[5],ymm4[5],ymm5[6],ymm4[6],ymm5[7],ymm4[7],ymm5[12],ymm4[12],ymm5[13],ymm4[13],ymm5[14],ymm4[14],ymm5[15],ymm4[15]3750; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm8[4],ymm7[4],ymm8[5],ymm7[5],ymm8[6],ymm7[6],ymm8[7],ymm7[7],ymm8[12],ymm7[12],ymm8[13],ymm7[13],ymm8[14],ymm7[14],ymm8[15],ymm7[15]3751; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm9 = ymm6[0],ymm3[0],ymm6[1],ymm3[1],ymm6[4],ymm3[4],ymm6[5],ymm3[5]3752; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,2,3]3753; AVX2-FP-NEXT:    vmovdqa (%rdi), %ymm133754; AVX2-FP-NEXT:    vmovdqa (%rsi), %ymm143755; AVX2-FP-NEXT:    vmovdqa (%rdx), %ymm153756; AVX2-FP-NEXT:    vmovdqa (%rcx), %ymm23757; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm15[4],ymm2[4],ymm15[5],ymm2[5],ymm15[6],ymm2[6],ymm15[7],ymm2[7],ymm15[12],ymm2[12],ymm15[13],ymm2[13],ymm15[14],ymm2[14],ymm15[15],ymm2[15]3758; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm13[4],ymm14[4],ymm13[5],ymm14[5],ymm13[6],ymm14[6],ymm13[7],ymm14[7],ymm13[12],ymm14[12],ymm13[13],ymm14[13],ymm13[14],ymm14[14],ymm13[15],ymm14[15]3759; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm12 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]3760; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,1,3,3]3761; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0,1],ymm9[2,3],ymm12[4,5],ymm9[6,7]3762; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm3 = ymm6[2],ymm3[2],ymm6[3],ymm3[3],ymm6[6],ymm3[6],ymm6[7],ymm3[7]3763; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]3764; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm3[0,2,2,3]3765; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3]3766; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]3767; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[8],ymm4[8],ymm5[9],ymm4[9],ymm5[10],ymm4[10],ymm5[11],ymm4[11]3768; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm8[0],ymm7[0],ymm8[1],ymm7[1],ymm8[2],ymm7[2],ymm8[3],ymm7[3],ymm8[8],ymm7[8],ymm8[9],ymm7[9],ymm8[10],ymm7[10],ymm8[11],ymm7[11]3769; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm15[0],ymm2[0],ymm15[1],ymm2[1],ymm15[2],ymm2[2],ymm15[3],ymm2[3],ymm15[8],ymm2[8],ymm15[9],ymm2[9],ymm15[10],ymm2[10],ymm15[11],ymm2[11]3770; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm13[0],ymm14[0],ymm13[1],ymm14[1],ymm13[2],ymm14[2],ymm13[3],ymm14[3],ymm13[8],ymm14[8],ymm13[9],ymm14[9],ymm13[10],ymm14[10],ymm13[11],ymm14[11]3771; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm5 = ymm3[0],ymm1[0],ymm3[1],ymm1[1],ymm3[4],ymm1[4],ymm3[5],ymm1[5]3772; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,2,2,3]3773; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm6 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[4],ymm2[4],ymm4[5],ymm2[5]3774; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,1,3,3]3775; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0,1],ymm5[2,3],ymm6[4,5],ymm5[6,7]3776; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm1 = ymm3[2],ymm1[2],ymm3[3],ymm1[3],ymm3[6],ymm1[6],ymm3[7],ymm1[7]3777; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm4[2],ymm2[2],ymm4[3],ymm2[3],ymm4[6],ymm2[6],ymm4[7],ymm2[7]3778; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]3779; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,1,3,3]3780; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5],ymm1[6,7]3781; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %rax3782; AVX2-FP-NEXT:    vmovdqa %ymm1, 160(%rax)3783; AVX2-FP-NEXT:    vmovdqa %ymm5, 128(%rax)3784; AVX2-FP-NEXT:    vmovdqa %ymm0, 224(%rax)3785; AVX2-FP-NEXT:    vmovdqa %ymm9, 192(%rax)3786; AVX2-FP-NEXT:    vmovdqa %ymm11, 416(%rax)3787; AVX2-FP-NEXT:    vmovdqa %ymm10, 384(%rax)3788; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3789; AVX2-FP-NEXT:    vmovaps %ymm0, 480(%rax)3790; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3791; AVX2-FP-NEXT:    vmovaps %ymm0, 448(%rax)3792; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3793; AVX2-FP-NEXT:    vmovaps %ymm0, 96(%rax)3794; AVX2-FP-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload3795; AVX2-FP-NEXT:    vmovaps %ymm0, 64(%rax)3796; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3797; AVX2-FP-NEXT:    vmovaps %ymm0, 288(%rax)3798; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3799; AVX2-FP-NEXT:    vmovaps %ymm0, 256(%rax)3800; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3801; AVX2-FP-NEXT:    vmovaps %ymm0, 352(%rax)3802; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3803; AVX2-FP-NEXT:    vmovaps %ymm0, 320(%rax)3804; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3805; AVX2-FP-NEXT:    vmovaps %ymm0, (%rax)3806; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload3807; AVX2-FP-NEXT:    vmovaps %ymm0, 32(%rax)3808; AVX2-FP-NEXT:    addq $232, %rsp3809; AVX2-FP-NEXT:    vzeroupper3810; AVX2-FP-NEXT:    retq3811;3812; AVX2-FCP-LABEL: store_i16_stride8_vf32:3813; AVX2-FCP:       # %bb.0:3814; AVX2-FCP-NEXT:    subq $296, %rsp # imm = 0x1283815; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax3816; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r103817; AVX2-FCP-NEXT:    vmovdqa (%rax), %xmm03818; AVX2-FCP-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3819; AVX2-FCP-NEXT:    vmovdqa (%r10), %xmm13820; AVX2-FCP-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3821; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]3822; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm4 = [2,2,2,2,0,0,3,3]3823; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm4, %ymm03824; AVX2-FCP-NEXT:    vmovdqa (%r9), %xmm33825; AVX2-FCP-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3826; AVX2-FCP-NEXT:    vmovdqa (%r8), %xmm23827; AVX2-FCP-NEXT:    vmovdqa %xmm2, (%rsp) # 16-byte Spill3828; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]3829; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm4, %ymm33830; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3],ymm3[4,5,6],ymm0[7]3831; AVX2-FCP-NEXT:    vmovdqa (%rcx), %xmm43832; AVX2-FCP-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3833; AVX2-FCP-NEXT:    vmovdqa (%rdx), %xmm33834; AVX2-FCP-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3835; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]3836; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm7 = [2,2,3,3,3,3,0,0]3837; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm7, %ymm43838; AVX2-FCP-NEXT:    vmovdqa (%rsi), %xmm63839; AVX2-FCP-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3840; AVX2-FCP-NEXT:    vmovdqa (%rdi), %xmm53841; AVX2-FCP-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill3842; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]3843; AVX2-FCP-NEXT:    vpermd %ymm5, %ymm7, %ymm63844; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm6[0],ymm4[1],ymm6[2,3,4],ymm4[5],ymm6[6,7]3845; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1],ymm0[2,3],ymm4[4,5],ymm0[6,7]3846; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3847; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [0,0,0,0,0,0,1,1]3848; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm0, %ymm43849; AVX2-FCP-NEXT:    vmovdqa %ymm0, %ymm13850; AVX2-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm7 = [0,0,0,1]3851; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm7, %ymm23852; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm2[0,1,2],ymm4[3],ymm2[4,5,6],ymm4[7]3853; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [0,0,1,1,1,1,0,0]3854; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm0, %ymm63855; AVX2-FCP-NEXT:    vmovdqa %ymm0, %ymm33856; AVX2-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm14 = [0,1,1,0]3857; AVX2-FCP-NEXT:    vpermd %ymm5, %ymm14, %ymm53858; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0],ymm6[1],ymm5[2,3,4],ymm6[5],ymm5[6,7]3859; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1],ymm4[2,3],ymm5[4,5],ymm4[6,7]3860; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3861; AVX2-FCP-NEXT:    vmovdqa 32(%rax), %xmm93862; AVX2-FCP-NEXT:    vmovdqa 32(%r10), %xmm83863; AVX2-FCP-NEXT:    vmovdqa 32(%r9), %xmm133864; AVX2-FCP-NEXT:    vmovdqa 32(%r8), %xmm103865; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm8[4],xmm9[4],xmm8[5],xmm9[5],xmm8[6],xmm9[6],xmm8[7],xmm9[7]3866; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm1, %ymm03867; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm10[4],xmm13[4],xmm10[5],xmm13[5],xmm10[6],xmm13[6],xmm10[7],xmm13[7]3868; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm7, %ymm113869; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm12 = ymm11[0,1,2],ymm0[3],ymm11[4,5,6],ymm0[7]3870; AVX2-FCP-NEXT:    vmovdqa 32(%rcx), %xmm113871; AVX2-FCP-NEXT:    vmovdqa 32(%rdx), %xmm53872; AVX2-FCP-NEXT:    vmovdqa 32(%rsi), %xmm73873; AVX2-FCP-NEXT:    vmovdqa 32(%rdi), %xmm63874; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm11[4],xmm5[5],xmm11[5],xmm5[6],xmm11[6],xmm5[7],xmm11[7]3875; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm33876; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7]3877; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm14, %ymm143878; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm14[0],ymm3[1],ymm14[2,3,4],ymm3[5],ymm14[6,7]3879; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm12[2,3],ymm3[4,5],ymm12[6,7]3880; AVX2-FCP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3881; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm12 = [2,2,2,2,0,0,3,3]3882; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm12, %ymm33883; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm12, %ymm23884; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3],ymm2[4,5,6],ymm3[7]3885; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm15 = [2,2,3,3,3,3,0,0]3886; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm15, %ymm13887; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm15, %ymm03888; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4],ymm1[5],ymm0[6,7]3889; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2,3],ymm0[4,5],ymm2[6,7]3890; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3891; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]3892; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm10[0],xmm13[0],xmm10[1],xmm13[1],xmm10[2],xmm13[2],xmm10[3],xmm13[3]3893; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm8 = [0,0,0,0,0,0,1,1]3894; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm8, %ymm23895; AVX2-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm10 = [0,0,0,1]3896; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm10, %ymm33897; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]3898; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm11[0],xmm5[1],xmm11[1],xmm5[2],xmm11[2],xmm5[3],xmm11[3]3899; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]3900; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm11 = [0,0,1,1,1,1,0,0]3901; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm11, %ymm53902; AVX2-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm7 = [0,1,1,0]3903; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm7, %ymm63904; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3,4],ymm5[5],ymm6[6,7]3905; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm2[2,3],ymm5[4,5],ymm2[6,7]3906; AVX2-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3907; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm12, %ymm03908; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm12, %ymm13909; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6],ymm0[7]3910; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm15, %ymm13911; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm15, %ymm23912; AVX2-FCP-NEXT:    vmovdqa %ymm15, %ymm93913; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3,4],ymm1[5],ymm2[6,7]3914; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]3915; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3916; AVX2-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload3917; AVX2-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload3918; AVX2-FCP-NEXT:    # xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7]3919; AVX2-FCP-NEXT:    vmovdqa (%rsp), %xmm1 # 16-byte Reload3920; AVX2-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm2 # 16-byte Folded Reload3921; AVX2-FCP-NEXT:    # xmm2 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]3922; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm8, %ymm13923; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm10, %ymm33924; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5,6],ymm1[7]3925; AVX2-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload3926; AVX2-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload3927; AVX2-FCP-NEXT:    # xmm3 = xmm3[4],mem[4],xmm3[5],mem[5],xmm3[6],mem[6],xmm3[7],mem[7]3928; AVX2-FCP-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload3929; AVX2-FCP-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload3930; AVX2-FCP-NEXT:    # xmm4 = xmm4[4],mem[4],xmm4[5],mem[5],xmm4[6],mem[6],xmm4[7],mem[7]3931; AVX2-FCP-NEXT:    vmovdqa 32(%r8), %ymm143932; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm11, %ymm53933; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm7, %ymm63934; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3,4],ymm5[5],ymm6[6,7]3935; AVX2-FCP-NEXT:    vmovdqa 32(%r9), %ymm153936; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1],ymm1[2,3],ymm5[4,5],ymm1[6,7]3937; AVX2-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3938; AVX2-FCP-NEXT:    vmovdqa 32(%r10), %ymm83939; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm12, %ymm53940; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm12, %ymm23941; AVX2-FCP-NEXT:    vmovdqa 32(%rax), %ymm73942; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2],ymm5[3],ymm2[4,5,6],ymm5[7]3943; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm9, %ymm33944; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm9, %ymm43945; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3,4],ymm3[5],ymm4[6,7]3946; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7]3947; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3948; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm8[4],ymm7[4],ymm8[5],ymm7[5],ymm8[6],ymm7[6],ymm8[7],ymm7[7],ymm8[12],ymm7[12],ymm8[13],ymm7[13],ymm8[14],ymm7[14],ymm8[15],ymm7[15]3949; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [0,0,4,4,4,4,6,5]3950; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm0, %ymm33951; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm11 = ymm14[4],ymm15[4],ymm14[5],ymm15[5],ymm14[6],ymm15[6],ymm14[7],ymm15[7],ymm14[12],ymm15[12],ymm14[13],ymm15[13],ymm14[14],ymm15[14],ymm14[15],ymm15[15]3952; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [0,1,4,5,4,5,5,7]3953; AVX2-FCP-NEXT:    vpermd %ymm11, %ymm0, %ymm43954; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm4[0,1,2],ymm3[3],ymm4[4,5,6],ymm3[7]3955; AVX2-FCP-NEXT:    vmovdqa 32(%rdi), %ymm93956; AVX2-FCP-NEXT:    vmovdqa 32(%rsi), %ymm63957; AVX2-FCP-NEXT:    vmovdqa 32(%rdx), %ymm33958; AVX2-FCP-NEXT:    vmovdqa 32(%rcx), %ymm13959; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm13 = ymm3[4],ymm1[4],ymm3[5],ymm1[5],ymm3[6],ymm1[6],ymm3[7],ymm1[7],ymm3[12],ymm1[12],ymm3[13],ymm1[13],ymm3[14],ymm1[14],ymm3[15],ymm1[15]3960; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [4,4,2,1,6,5,6,5]3961; AVX2-FCP-NEXT:    vpermd %ymm13, %ymm0, %ymm123962; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm9[4],ymm6[4],ymm9[5],ymm6[5],ymm9[6],ymm6[6],ymm9[7],ymm6[7],ymm9[12],ymm6[12],ymm9[13],ymm6[13],ymm9[14],ymm6[14],ymm9[15],ymm6[15]3963; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm4 = [4,5,1,3,5,7,5,7]3964; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm4, %ymm103965; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0],ymm12[1],ymm10[2,3,4],ymm12[5],ymm10[6,7]3966; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm10[0,1],ymm5[2,3],ymm10[4,5],ymm5[6,7]3967; AVX2-FCP-NEXT:    vmovdqu %ymm5, (%rsp) # 32-byte Spill3968; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm4 = [0,2,4,6,4,6,6,7]3969; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm4, %ymm23970; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm4 = [2,1,6,5,6,5,7,7]3971; AVX2-FCP-NEXT:    vpermd %ymm11, %ymm4, %ymm103972; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2],ymm2[3],ymm10[4,5,6],ymm2[7]3973; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm5 = [4,6,2,3,6,7,6,7]3974; AVX2-FCP-NEXT:    vpermd %ymm13, %ymm5, %ymm113975; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm13 = [6,5,3,3,7,7,7,7]3976; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm13, %ymm03977; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm11[1],ymm0[2,3,4],ymm11[5],ymm0[6,7]3978; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm10[2,3],ymm0[4,5],ymm10[6,7]3979; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3980; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm8[0],ymm7[0],ymm8[1],ymm7[1],ymm8[2],ymm7[2],ymm8[3],ymm7[3],ymm8[8],ymm7[8],ymm8[9],ymm7[9],ymm8[10],ymm7[10],ymm8[11],ymm7[11]3981; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm14[0],ymm15[0],ymm14[1],ymm15[1],ymm14[2],ymm15[2],ymm14[3],ymm15[3],ymm14[8],ymm15[8],ymm14[9],ymm15[9],ymm14[10],ymm15[10],ymm14[11],ymm15[11]3982; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm2 = [0,0,4,4,4,4,6,5]3983; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm2, %ymm83984; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm12 = [0,1,4,5,4,5,5,7]3985; AVX2-FCP-NEXT:    vpermd %ymm7, %ymm12, %ymm103986; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm8 = ymm10[0,1,2],ymm8[3],ymm10[4,5,6],ymm8[7]3987; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm3[0],ymm1[0],ymm3[1],ymm1[1],ymm3[2],ymm1[2],ymm3[3],ymm1[3],ymm3[8],ymm1[8],ymm3[9],ymm1[9],ymm3[10],ymm1[10],ymm3[11],ymm1[11]3988; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm9[0],ymm6[0],ymm9[1],ymm6[1],ymm9[2],ymm6[2],ymm9[3],ymm6[3],ymm9[8],ymm6[8],ymm9[9],ymm6[9],ymm9[10],ymm6[10],ymm9[11],ymm6[11]3989; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm14 = [4,4,2,1,6,5,6,5]3990; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm14, %ymm43991; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm15 = [4,5,1,3,5,7,5,7]3992; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm15, %ymm63993; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm6[0],ymm4[1],ymm6[2,3,4],ymm4[5],ymm6[6,7]3994; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm8[2,3],ymm4[4,5],ymm8[6,7]3995; AVX2-FCP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill3996; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm4 = [0,2,4,6,4,6,6,7]3997; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm4, %ymm03998; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm4 = [2,1,6,5,6,5,7,7]3999; AVX2-FCP-NEXT:    vpermd %ymm7, %ymm4, %ymm44000; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2],ymm0[3],ymm4[4,5,6],ymm0[7]4001; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm5, %ymm14002; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm13, %ymm34003; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2,3,4],ymm1[5],ymm3[6,7]4004; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]4005; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4006; AVX2-FCP-NEXT:    vmovdqa (%r8), %ymm134007; AVX2-FCP-NEXT:    vmovdqa (%r9), %ymm114008; AVX2-FCP-NEXT:    vmovdqa (%r10), %ymm94009; AVX2-FCP-NEXT:    vmovdqa (%rax), %ymm54010; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm9[4],ymm5[4],ymm9[5],ymm5[5],ymm9[6],ymm5[6],ymm9[7],ymm5[7],ymm9[12],ymm5[12],ymm9[13],ymm5[13],ymm9[14],ymm5[14],ymm9[15],ymm5[15]4011; AVX2-FCP-NEXT:    vpermd %ymm6, %ymm2, %ymm74012; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm8 = ymm13[4],ymm11[4],ymm13[5],ymm11[5],ymm13[6],ymm11[6],ymm13[7],ymm11[7],ymm13[12],ymm11[12],ymm13[13],ymm11[13],ymm13[14],ymm11[14],ymm13[15],ymm11[15]4013; AVX2-FCP-NEXT:    vpermd %ymm8, %ymm12, %ymm104014; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm12 = ymm10[0,1,2],ymm7[3],ymm10[4,5,6],ymm7[7]4015; AVX2-FCP-NEXT:    vmovdqa (%rdi), %ymm104016; AVX2-FCP-NEXT:    vmovdqa (%rsi), %ymm74017; AVX2-FCP-NEXT:    vmovdqa (%rdx), %ymm44018; AVX2-FCP-NEXT:    vmovdqa (%rcx), %ymm34019; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]4020; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm14, %ymm24021; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm10[4],ymm7[4],ymm10[5],ymm7[5],ymm10[6],ymm7[6],ymm10[7],ymm7[7],ymm10[12],ymm7[12],ymm10[13],ymm7[13],ymm10[14],ymm7[14],ymm10[15],ymm7[15]4022; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm15, %ymm144023; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm14[0],ymm2[1],ymm14[2,3,4],ymm2[5],ymm14[6,7]4024; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm12[2,3],ymm2[4,5],ymm12[6,7]4025; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm12 = [0,2,4,6,4,6,6,7]4026; AVX2-FCP-NEXT:    vpermd %ymm6, %ymm12, %ymm64027; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm14 = [2,1,6,5,6,5,7,7]4028; AVX2-FCP-NEXT:    vpermd %ymm8, %ymm14, %ymm84029; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2],ymm6[3],ymm8[4,5,6],ymm6[7]4030; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm15 = [4,6,2,3,6,7,6,7]4031; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm15, %ymm14032; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm8 = [6,5,3,3,7,7,7,7]4033; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm8, %ymm04034; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4],ymm1[5],ymm0[6,7]4035; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm6[2,3],ymm0[4,5],ymm6[6,7]4036; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm9[0],ymm5[0],ymm9[1],ymm5[1],ymm9[2],ymm5[2],ymm9[3],ymm5[3],ymm9[8],ymm5[8],ymm9[9],ymm5[9],ymm9[10],ymm5[10],ymm9[11],ymm5[11]4037; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[8],ymm11[8],ymm13[9],ymm11[9],ymm13[10],ymm11[10],ymm13[11],ymm11[11]4038; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm6 = [0,0,4,4,4,4,6,5]4039; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm6, %ymm64040; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm8 = [0,1,4,5,4,5,5,7]4041; AVX2-FCP-NEXT:    vpermd %ymm5, %ymm8, %ymm84042; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2],ymm6[3],ymm8[4,5,6],ymm6[7]4043; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]4044; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm10[0],ymm7[0],ymm10[1],ymm7[1],ymm10[2],ymm7[2],ymm10[3],ymm7[3],ymm10[8],ymm7[8],ymm10[9],ymm7[9],ymm10[10],ymm7[10],ymm10[11],ymm7[11]4045; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm7 = [4,4,2,1,6,5,6,5]4046; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm7, %ymm74047; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm8 = [4,5,1,3,5,7,5,7]4048; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm8, %ymm84049; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm8[0],ymm7[1],ymm8[2,3,4],ymm7[5],ymm8[6,7]4050; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,3],ymm7[4,5],ymm6[6,7]4051; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm12, %ymm14052; AVX2-FCP-NEXT:    vpermd %ymm5, %ymm14, %ymm54053; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2],ymm1[3],ymm5[4,5,6],ymm1[7]4054; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm15, %ymm34055; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm5 = [6,5,3,3,7,7,7,7]4056; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm5, %ymm44057; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3,4],ymm3[5],ymm4[6,7]4058; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,3],ymm3[4,5],ymm1[6,7]4059; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax4060; AVX2-FCP-NEXT:    vmovdqa %ymm1, 160(%rax)4061; AVX2-FCP-NEXT:    vmovdqa %ymm6, 128(%rax)4062; AVX2-FCP-NEXT:    vmovdqa %ymm0, 224(%rax)4063; AVX2-FCP-NEXT:    vmovdqa %ymm2, 192(%rax)4064; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload4065; AVX2-FCP-NEXT:    vmovaps %ymm0, 416(%rax)4066; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload4067; AVX2-FCP-NEXT:    vmovaps %ymm0, 384(%rax)4068; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload4069; AVX2-FCP-NEXT:    vmovaps %ymm0, 480(%rax)4070; AVX2-FCP-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload4071; AVX2-FCP-NEXT:    vmovaps %ymm0, 448(%rax)4072; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload4073; AVX2-FCP-NEXT:    vmovaps %ymm0, 96(%rax)4074; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload4075; AVX2-FCP-NEXT:    vmovaps %ymm0, 64(%rax)4076; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload4077; AVX2-FCP-NEXT:    vmovaps %ymm0, 288(%rax)4078; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload4079; AVX2-FCP-NEXT:    vmovaps %ymm0, 256(%rax)4080; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload4081; AVX2-FCP-NEXT:    vmovaps %ymm0, 352(%rax)4082; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload4083; AVX2-FCP-NEXT:    vmovaps %ymm0, 320(%rax)4084; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload4085; AVX2-FCP-NEXT:    vmovaps %ymm0, (%rax)4086; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload4087; AVX2-FCP-NEXT:    vmovaps %ymm0, 32(%rax)4088; AVX2-FCP-NEXT:    addq $296, %rsp # imm = 0x1284089; AVX2-FCP-NEXT:    vzeroupper4090; AVX2-FCP-NEXT:    retq4091;4092; AVX512-LABEL: store_i16_stride8_vf32:4093; AVX512:       # %bb.0:4094; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax4095; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r104096; AVX512-NEXT:    vmovdqa (%r10), %xmm04097; AVX512-NEXT:    vmovdqa (%rax), %xmm14098; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]4099; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm204100; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm214101; AVX512-NEXT:    vmovdqa (%r9), %xmm04102; AVX512-NEXT:    vmovdqa (%r8), %xmm14103; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]4104; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm224105; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm234106; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]4107; AVX512-NEXT:    vpermt2d %zmm2, %zmm16, %zmm54108; AVX512-NEXT:    vmovdqa (%rcx), %xmm04109; AVX512-NEXT:    vmovdqa (%rdx), %xmm14110; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]4111; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm244112; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm254113; AVX512-NEXT:    vmovdqa (%rsi), %xmm04114; AVX512-NEXT:    vmovdqa (%rdi), %xmm14115; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]4116; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm264117; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm274118; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]4119; AVX512-NEXT:    vpermt2d %zmm10, %zmm17, %zmm94120; AVX512-NEXT:    movb $-86, %r11b4121; AVX512-NEXT:    kmovw %r11d, %k14122; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm9 {%k1}4123; AVX512-NEXT:    vmovdqa 32(%r10), %ymm54124; AVX512-NEXT:    vmovdqa 32(%rax), %ymm104125; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm10[0],ymm5[0],ymm10[1],ymm5[1],ymm10[2],ymm5[2],ymm10[3],ymm5[3],ymm10[8],ymm5[8],ymm10[9],ymm5[9],ymm10[10],ymm5[10],ymm10[11],ymm5[11]4126; AVX512-NEXT:    vmovdqa 32(%r9), %ymm134127; AVX512-NEXT:    vmovdqa 32(%r8), %ymm154128; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm15[0],ymm13[0],ymm15[1],ymm13[1],ymm15[2],ymm13[2],ymm15[3],ymm13[3],ymm15[8],ymm13[8],ymm15[9],ymm13[9],ymm15[10],ymm13[10],ymm15[11],ymm13[11]4129; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]4130; AVX512-NEXT:    vpermt2d %zmm11, %zmm18, %zmm04131; AVX512-NEXT:    vmovdqa 32(%rcx), %ymm34132; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm124133; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]4134; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm12[0],ymm3[0],ymm12[1],ymm3[1],ymm12[2],ymm3[2],ymm12[3],ymm3[3],ymm12[8],ymm3[8],ymm12[9],ymm3[9],ymm12[10],ymm3[10],ymm12[11],ymm3[11]4135; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm24136; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm14137; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]4138; AVX512-NEXT:    vpermt2d %zmm14, %zmm19, %zmm114139; AVX512-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k1}4140; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm10[4],ymm5[4],ymm10[5],ymm5[5],ymm10[6],ymm5[6],ymm10[7],ymm5[7],ymm10[12],ymm5[12],ymm10[13],ymm5[13],ymm10[14],ymm5[14],ymm10[15],ymm5[15]4141; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm15[4],ymm13[4],ymm15[5],ymm13[5],ymm15[6],ymm13[6],ymm15[7],ymm13[7],ymm15[12],ymm13[12],ymm15[13],ymm13[13],ymm15[14],ymm13[14],ymm15[15],ymm13[15]4142; AVX512-NEXT:    vpermt2d %zmm0, %zmm18, %zmm54143; AVX512-NEXT:    vmovdqa 32(%r10), %xmm64144; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm12[4],ymm3[4],ymm12[5],ymm3[5],ymm12[6],ymm3[6],ymm12[7],ymm3[7],ymm12[12],ymm3[12],ymm12[13],ymm3[13],ymm12[14],ymm3[14],ymm12[15],ymm3[15]4145; AVX512-NEXT:    vmovdqa 32(%rax), %xmm104146; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm13 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]4147; AVX512-NEXT:    vmovdqa 32(%r9), %xmm44148; AVX512-NEXT:    vpermt2d %zmm3, %zmm19, %zmm134149; AVX512-NEXT:    vmovdqa 32(%r8), %xmm24150; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm13 {%k1}4151; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]4152; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]4153; AVX512-NEXT:    vpermt2d %zmm3, %zmm16, %zmm54154; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm34155; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm124156; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm144157; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm04158; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm12[0],xmm3[0],xmm12[1],xmm3[1],xmm12[2],xmm3[2],xmm12[3],xmm3[3]4159; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]4160; AVX512-NEXT:    vpermt2d %zmm1, %zmm17, %zmm154161; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm15 {%k1}4162; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]4163; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]4164; AVX512-NEXT:    vpermt2d %zmm1, %zmm16, %zmm24165; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm12[4],xmm3[4],xmm12[5],xmm3[5],xmm12[6],xmm3[6],xmm12[7],xmm3[7]4166; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]4167; AVX512-NEXT:    vpermt2d %zmm1, %zmm17, %zmm54168; AVX512-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}4169; AVX512-NEXT:    vmovdqa (%r10), %ymm84170; AVX512-NEXT:    vmovdqa (%rax), %ymm74171; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]4172; AVX512-NEXT:    vmovdqa (%r9), %ymm34173; AVX512-NEXT:    vmovdqa (%r8), %ymm44174; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]4175; AVX512-NEXT:    vpermt2d %zmm2, %zmm18, %zmm64176; AVX512-NEXT:    vmovdqa (%rcx), %ymm24177; AVX512-NEXT:    vmovdqa (%rdx), %ymm124178; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm12[0],ymm2[0],ymm12[1],ymm2[1],ymm12[2],ymm2[2],ymm12[3],ymm2[3],ymm12[8],ymm2[8],ymm12[9],ymm2[9],ymm12[10],ymm2[10],ymm12[11],ymm2[11]4179; AVX512-NEXT:    vmovdqa (%rsi), %ymm14180; AVX512-NEXT:    vmovdqa (%rdi), %ymm04181; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]4182; AVX512-NEXT:    vpermt2d %zmm14, %zmm19, %zmm104183; AVX512-NEXT:    vmovdqa64 %zmm6, %zmm10 {%k1}4184; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15]4185; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]4186; AVX512-NEXT:    vpermt2d %zmm6, %zmm18, %zmm34187; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm12[4],ymm2[4],ymm12[5],ymm2[5],ymm12[6],ymm2[6],ymm12[7],ymm2[7],ymm12[12],ymm2[12],ymm12[13],ymm2[13],ymm12[14],ymm2[14],ymm12[15],ymm2[15]4188; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]4189; AVX512-NEXT:    vpermt2d %zmm2, %zmm19, %zmm04190; AVX512-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k1}4191; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm14192; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm24193; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]4194; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm24195; AVX512-NEXT:    vmovdqa64 %xmm23, %xmm34196; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]4197; AVX512-NEXT:    vpermt2d %zmm1, %zmm16, %zmm24198; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm14199; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm34200; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]4201; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm34202; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm44203; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]4204; AVX512-NEXT:    vpermt2d %zmm1, %zmm17, %zmm34205; AVX512-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}4206; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax4207; AVX512-NEXT:    vmovdqa64 %zmm3, (%rax)4208; AVX512-NEXT:    vmovdqa64 %zmm0, 192(%rax)4209; AVX512-NEXT:    vmovdqa64 %zmm10, 128(%rax)4210; AVX512-NEXT:    vmovdqa64 %zmm5, 320(%rax)4211; AVX512-NEXT:    vmovdqa64 %zmm15, 256(%rax)4212; AVX512-NEXT:    vmovdqa64 %zmm13, 448(%rax)4213; AVX512-NEXT:    vmovdqa64 %zmm11, 384(%rax)4214; AVX512-NEXT:    vmovdqa64 %zmm9, 64(%rax)4215; AVX512-NEXT:    vzeroupper4216; AVX512-NEXT:    retq4217;4218; AVX512-FCP-LABEL: store_i16_stride8_vf32:4219; AVX512-FCP:       # %bb.0:4220; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax4221; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r104222; AVX512-FCP-NEXT:    vmovdqa (%r10), %xmm14223; AVX512-FCP-NEXT:    vmovdqa (%rax), %xmm24224; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]4225; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm214226; AVX512-FCP-NEXT:    vmovdqa64 %xmm1, %xmm224227; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm14228; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm24229; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]4230; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm234231; AVX512-FCP-NEXT:    vmovdqa64 %xmm1, %xmm244232; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]4233; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm16, %zmm54234; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm14235; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm24236; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]4237; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm254238; AVX512-FCP-NEXT:    vmovdqa64 %xmm1, %xmm264239; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm14240; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm94241; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm9[4],xmm1[4],xmm9[5],xmm1[5],xmm9[6],xmm1[6],xmm9[7],xmm1[7]4242; AVX512-FCP-NEXT:    vmovdqa64 %xmm1, %xmm274243; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [0,16,1,0,1,17,0,0,2,18,3,0,3,19,0,0]4244; AVX512-FCP-NEXT:    vpermi2d %zmm0, %zmm10, %zmm194245; AVX512-FCP-NEXT:    movb $-86, %r11b4246; AVX512-FCP-NEXT:    kmovw %r11d, %k14247; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, %zmm19 {%k1}4248; AVX512-FCP-NEXT:    vmovdqa 32(%r10), %ymm14249; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %ymm54250; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm5[0],ymm1[0],ymm5[1],ymm1[1],ymm5[2],ymm1[2],ymm5[3],ymm1[3],ymm5[8],ymm1[8],ymm5[9],ymm1[9],ymm5[10],ymm1[10],ymm5[11],ymm1[11]4251; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %ymm124252; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm144253; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm14[0],ymm12[0],ymm14[1],ymm12[1],ymm14[2],ymm12[2],ymm14[3],ymm12[3],ymm14[8],ymm12[8],ymm14[9],ymm12[9],ymm14[10],ymm12[10],ymm14[11],ymm12[11]4254; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]4255; AVX512-FCP-NEXT:    vpermt2d %zmm10, %zmm17, %zmm154256; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm114257; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm134258; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[8],ymm11[8],ymm13[9],ymm11[9],ymm13[10],ymm11[10],ymm13[11],ymm11[11]4259; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]4260; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm34261; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm04262; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[8],ymm3[8],ymm0[9],ymm3[9],ymm0[10],ymm3[10],ymm0[11],ymm3[11]4263; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm18, %zmm104264; AVX512-FCP-NEXT:    vmovdqa64 %zmm15, %zmm10 {%k1}4265; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm5[4],ymm1[4],ymm5[5],ymm1[5],ymm5[6],ymm1[6],ymm5[7],ymm1[7],ymm5[12],ymm1[12],ymm5[13],ymm1[13],ymm5[14],ymm1[14],ymm5[15],ymm1[15]4266; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm14[4],ymm12[4],ymm14[5],ymm12[5],ymm14[6],ymm12[6],ymm14[7],ymm12[7],ymm14[12],ymm12[12],ymm14[13],ymm12[13],ymm14[14],ymm12[14],ymm14[15],ymm12[15]4267; AVX512-FCP-NEXT:    vmovdqa 32(%r10), %xmm64268; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm54269; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %xmm44270; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm11 = ymm13[4],ymm11[4],ymm13[5],ymm11[5],ymm13[6],ymm11[6],ymm13[7],ymm11[7],ymm13[12],ymm11[12],ymm13[13],ymm11[13],ymm13[14],ymm11[14],ymm13[15],ymm11[15]4271; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm134272; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm12 = ymm0[4],ymm3[4],ymm0[5],ymm3[5],ymm0[6],ymm3[6],ymm0[7],ymm3[7],ymm0[12],ymm3[12],ymm0[13],ymm3[13],ymm0[14],ymm3[14],ymm0[15],ymm3[15]4273; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm24274; AVX512-FCP-NEXT:    vpermt2d %zmm11, %zmm18, %zmm124275; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, %zmm12 {%k1}4276; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]4277; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm13[0],xmm2[1],xmm13[1],xmm2[2],xmm13[2],xmm2[3],xmm13[3]4278; AVX512-FCP-NEXT:    vpermt2d %zmm3, %zmm16, %zmm54279; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm34280; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]4281; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm114282; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm154283; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm04284; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm11[0],xmm3[0],xmm11[1],xmm3[1],xmm11[2],xmm3[2],xmm11[3],xmm3[3]4285; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]4286; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm20, %zmm144287; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, %zmm14 {%k1}4288; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]4289; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm13[4],xmm2[5],xmm13[5],xmm2[6],xmm13[6],xmm2[7],xmm13[7]4290; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm16, %zmm24291; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm3[4],xmm11[5],xmm3[5],xmm11[6],xmm3[6],xmm11[7],xmm3[7]4292; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm0[4],xmm15[4],xmm0[5],xmm15[5],xmm0[6],xmm15[6],xmm0[7],xmm15[7]4293; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm20, %zmm54294; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}4295; AVX512-FCP-NEXT:    vmovdqa (%r10), %ymm84296; AVX512-FCP-NEXT:    vmovdqa (%rax), %ymm74297; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]4298; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm34299; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm44300; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]4301; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm17, %zmm64302; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm114303; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm134304; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[8],ymm11[8],ymm13[9],ymm11[9],ymm13[10],ymm11[10],ymm13[11],ymm11[11]4305; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm24306; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm14307; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]4308; AVX512-FCP-NEXT:    vpermt2d %zmm15, %zmm18, %zmm04309; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, %zmm0 {%k1}4310; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15]4311; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]4312; AVX512-FCP-NEXT:    vpermt2d %zmm6, %zmm17, %zmm34313; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm13[4],ymm11[4],ymm13[5],ymm11[5],ymm13[6],ymm11[6],ymm13[7],ymm11[7],ymm13[12],ymm11[12],ymm13[13],ymm11[13],ymm13[14],ymm11[14],ymm13[15],ymm11[15]4314; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]4315; AVX512-FCP-NEXT:    vpermt2d %zmm4, %zmm18, %zmm14316; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k1}4317; AVX512-FCP-NEXT:    vmovdqa64 %xmm21, %xmm24318; AVX512-FCP-NEXT:    vmovdqa64 %xmm22, %xmm34319; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]4320; AVX512-FCP-NEXT:    vmovdqa64 %xmm23, %xmm34321; AVX512-FCP-NEXT:    vmovdqa64 %xmm24, %xmm44322; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]4323; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm16, %zmm34324; AVX512-FCP-NEXT:    vmovdqa64 %xmm25, %xmm24325; AVX512-FCP-NEXT:    vmovdqa64 %xmm26, %xmm44326; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]4327; AVX512-FCP-NEXT:    vmovdqa64 %xmm27, %xmm44328; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]4329; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm20, %zmm44330; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}4331; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax4332; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)4333; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 192(%rax)4334; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 128(%rax)4335; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, 320(%rax)4336; AVX512-FCP-NEXT:    vmovdqa64 %zmm14, 256(%rax)4337; AVX512-FCP-NEXT:    vmovdqa64 %zmm12, 448(%rax)4338; AVX512-FCP-NEXT:    vmovdqa64 %zmm10, 384(%rax)4339; AVX512-FCP-NEXT:    vmovdqa64 %zmm19, 64(%rax)4340; AVX512-FCP-NEXT:    vzeroupper4341; AVX512-FCP-NEXT:    retq4342;4343; AVX512DQ-LABEL: store_i16_stride8_vf32:4344; AVX512DQ:       # %bb.0:4345; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax4346; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %r104347; AVX512DQ-NEXT:    vmovdqa (%r10), %xmm04348; AVX512DQ-NEXT:    vmovdqa (%rax), %xmm14349; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]4350; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm204351; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm214352; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm04353; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm14354; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]4355; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm224356; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm234357; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]4358; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm16, %zmm54359; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm04360; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm14361; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]4362; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm244363; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm254364; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm04365; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm14366; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]4367; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm264368; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm274369; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]4370; AVX512DQ-NEXT:    vpermt2d %zmm10, %zmm17, %zmm94371; AVX512DQ-NEXT:    movb $-86, %r11b4372; AVX512DQ-NEXT:    kmovw %r11d, %k14373; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm9 {%k1}4374; AVX512DQ-NEXT:    vmovdqa 32(%r10), %ymm54375; AVX512DQ-NEXT:    vmovdqa 32(%rax), %ymm104376; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm10[0],ymm5[0],ymm10[1],ymm5[1],ymm10[2],ymm5[2],ymm10[3],ymm5[3],ymm10[8],ymm5[8],ymm10[9],ymm5[9],ymm10[10],ymm5[10],ymm10[11],ymm5[11]4377; AVX512DQ-NEXT:    vmovdqa 32(%r9), %ymm134378; AVX512DQ-NEXT:    vmovdqa 32(%r8), %ymm154379; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm15[0],ymm13[0],ymm15[1],ymm13[1],ymm15[2],ymm13[2],ymm15[3],ymm13[3],ymm15[8],ymm13[8],ymm15[9],ymm13[9],ymm15[10],ymm13[10],ymm15[11],ymm13[11]4380; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]4381; AVX512DQ-NEXT:    vpermt2d %zmm11, %zmm18, %zmm04382; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %ymm34383; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm124384; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]4385; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm12[0],ymm3[0],ymm12[1],ymm3[1],ymm12[2],ymm3[2],ymm12[3],ymm3[3],ymm12[8],ymm3[8],ymm12[9],ymm3[9],ymm12[10],ymm3[10],ymm12[11],ymm3[11]4386; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm24387; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm14388; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]4389; AVX512DQ-NEXT:    vpermt2d %zmm14, %zmm19, %zmm114390; AVX512DQ-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k1}4391; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm10[4],ymm5[4],ymm10[5],ymm5[5],ymm10[6],ymm5[6],ymm10[7],ymm5[7],ymm10[12],ymm5[12],ymm10[13],ymm5[13],ymm10[14],ymm5[14],ymm10[15],ymm5[15]4392; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm15[4],ymm13[4],ymm15[5],ymm13[5],ymm15[6],ymm13[6],ymm15[7],ymm13[7],ymm15[12],ymm13[12],ymm15[13],ymm13[13],ymm15[14],ymm13[14],ymm15[15],ymm13[15]4393; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm18, %zmm54394; AVX512DQ-NEXT:    vmovdqa 32(%r10), %xmm64395; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm12[4],ymm3[4],ymm12[5],ymm3[5],ymm12[6],ymm3[6],ymm12[7],ymm3[7],ymm12[12],ymm3[12],ymm12[13],ymm3[13],ymm12[14],ymm3[14],ymm12[15],ymm3[15]4396; AVX512DQ-NEXT:    vmovdqa 32(%rax), %xmm104397; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm13 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]4398; AVX512DQ-NEXT:    vmovdqa 32(%r9), %xmm44399; AVX512DQ-NEXT:    vpermt2d %zmm3, %zmm19, %zmm134400; AVX512DQ-NEXT:    vmovdqa 32(%r8), %xmm24401; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm13 {%k1}4402; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]4403; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]4404; AVX512DQ-NEXT:    vpermt2d %zmm3, %zmm16, %zmm54405; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm34406; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm124407; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm144408; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm04409; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm12[0],xmm3[0],xmm12[1],xmm3[1],xmm12[2],xmm3[2],xmm12[3],xmm3[3]4410; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]4411; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm17, %zmm154412; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm15 {%k1}4413; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]4414; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]4415; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm16, %zmm24416; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm12[4],xmm3[4],xmm12[5],xmm3[5],xmm12[6],xmm3[6],xmm12[7],xmm3[7]4417; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]4418; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm17, %zmm54419; AVX512DQ-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}4420; AVX512DQ-NEXT:    vmovdqa (%r10), %ymm84421; AVX512DQ-NEXT:    vmovdqa (%rax), %ymm74422; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]4423; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm34424; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm44425; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]4426; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm18, %zmm64427; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm24428; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm124429; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm12[0],ymm2[0],ymm12[1],ymm2[1],ymm12[2],ymm2[2],ymm12[3],ymm2[3],ymm12[8],ymm2[8],ymm12[9],ymm2[9],ymm12[10],ymm2[10],ymm12[11],ymm2[11]4430; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm14431; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm04432; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]4433; AVX512DQ-NEXT:    vpermt2d %zmm14, %zmm19, %zmm104434; AVX512DQ-NEXT:    vmovdqa64 %zmm6, %zmm10 {%k1}4435; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15]4436; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]4437; AVX512DQ-NEXT:    vpermt2d %zmm6, %zmm18, %zmm34438; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm12[4],ymm2[4],ymm12[5],ymm2[5],ymm12[6],ymm2[6],ymm12[7],ymm2[7],ymm12[12],ymm2[12],ymm12[13],ymm2[13],ymm12[14],ymm2[14],ymm12[15],ymm2[15]4439; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]4440; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm19, %zmm04441; AVX512DQ-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k1}4442; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm14443; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm24444; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]4445; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm24446; AVX512DQ-NEXT:    vmovdqa64 %xmm23, %xmm34447; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]4448; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm16, %zmm24449; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm14450; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm34451; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]4452; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm34453; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm44454; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]4455; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm17, %zmm34456; AVX512DQ-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}4457; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax4458; AVX512DQ-NEXT:    vmovdqa64 %zmm3, (%rax)4459; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 192(%rax)4460; AVX512DQ-NEXT:    vmovdqa64 %zmm10, 128(%rax)4461; AVX512DQ-NEXT:    vmovdqa64 %zmm5, 320(%rax)4462; AVX512DQ-NEXT:    vmovdqa64 %zmm15, 256(%rax)4463; AVX512DQ-NEXT:    vmovdqa64 %zmm13, 448(%rax)4464; AVX512DQ-NEXT:    vmovdqa64 %zmm11, 384(%rax)4465; AVX512DQ-NEXT:    vmovdqa64 %zmm9, 64(%rax)4466; AVX512DQ-NEXT:    vzeroupper4467; AVX512DQ-NEXT:    retq4468;4469; AVX512DQ-FCP-LABEL: store_i16_stride8_vf32:4470; AVX512DQ-FCP:       # %bb.0:4471; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax4472; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r104473; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %xmm14474; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %xmm24475; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]4476; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm214477; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm224478; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm14479; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm24480; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]4481; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm234482; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm244483; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]4484; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm16, %zmm54485; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm14486; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm24487; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]4488; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm254489; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm264490; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm14491; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm94492; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm9[4],xmm1[4],xmm9[5],xmm1[5],xmm9[6],xmm1[6],xmm9[7],xmm1[7]4493; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm274494; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [0,16,1,0,1,17,0,0,2,18,3,0,3,19,0,0]4495; AVX512DQ-FCP-NEXT:    vpermi2d %zmm0, %zmm10, %zmm194496; AVX512DQ-FCP-NEXT:    movb $-86, %r11b4497; AVX512DQ-FCP-NEXT:    kmovw %r11d, %k14498; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, %zmm19 {%k1}4499; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r10), %ymm14500; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %ymm54501; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm5[0],ymm1[0],ymm5[1],ymm1[1],ymm5[2],ymm1[2],ymm5[3],ymm1[3],ymm5[8],ymm1[8],ymm5[9],ymm1[9],ymm5[10],ymm1[10],ymm5[11],ymm1[11]4502; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %ymm124503; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm144504; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm14[0],ymm12[0],ymm14[1],ymm12[1],ymm14[2],ymm12[2],ymm14[3],ymm12[3],ymm14[8],ymm12[8],ymm14[9],ymm12[9],ymm14[10],ymm12[10],ymm14[11],ymm12[11]4505; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]4506; AVX512DQ-FCP-NEXT:    vpermt2d %zmm10, %zmm17, %zmm154507; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm114508; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm134509; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[8],ymm11[8],ymm13[9],ymm11[9],ymm13[10],ymm11[10],ymm13[11],ymm11[11]4510; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]4511; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm34512; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm04513; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[8],ymm3[8],ymm0[9],ymm3[9],ymm0[10],ymm3[10],ymm0[11],ymm3[11]4514; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm18, %zmm104515; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm15, %zmm10 {%k1}4516; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm5[4],ymm1[4],ymm5[5],ymm1[5],ymm5[6],ymm1[6],ymm5[7],ymm1[7],ymm5[12],ymm1[12],ymm5[13],ymm1[13],ymm5[14],ymm1[14],ymm5[15],ymm1[15]4517; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm14[4],ymm12[4],ymm14[5],ymm12[5],ymm14[6],ymm12[6],ymm14[7],ymm12[7],ymm14[12],ymm12[12],ymm14[13],ymm12[13],ymm14[14],ymm12[14],ymm14[15],ymm12[15]4518; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r10), %xmm64519; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm54520; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %xmm44521; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm11 = ymm13[4],ymm11[4],ymm13[5],ymm11[5],ymm13[6],ymm11[6],ymm13[7],ymm11[7],ymm13[12],ymm11[12],ymm13[13],ymm11[13],ymm13[14],ymm11[14],ymm13[15],ymm11[15]4522; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm134523; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm12 = ymm0[4],ymm3[4],ymm0[5],ymm3[5],ymm0[6],ymm3[6],ymm0[7],ymm3[7],ymm0[12],ymm3[12],ymm0[13],ymm3[13],ymm0[14],ymm3[14],ymm0[15],ymm3[15]4524; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm24525; AVX512DQ-FCP-NEXT:    vpermt2d %zmm11, %zmm18, %zmm124526; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, %zmm12 {%k1}4527; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]4528; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm13[0],xmm2[1],xmm13[1],xmm2[2],xmm13[2],xmm2[3],xmm13[3]4529; AVX512DQ-FCP-NEXT:    vpermt2d %zmm3, %zmm16, %zmm54530; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm34531; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]4532; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm114533; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm154534; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm04535; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm11[0],xmm3[0],xmm11[1],xmm3[1],xmm11[2],xmm3[2],xmm11[3],xmm3[3]4536; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]4537; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm20, %zmm144538; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, %zmm14 {%k1}4539; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]4540; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm13[4],xmm2[5],xmm13[5],xmm2[6],xmm13[6],xmm2[7],xmm13[7]4541; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm16, %zmm24542; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm3[4],xmm11[5],xmm3[5],xmm11[6],xmm3[6],xmm11[7],xmm3[7]4543; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm0[4],xmm15[4],xmm0[5],xmm15[5],xmm0[6],xmm15[6],xmm0[7],xmm15[7]4544; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm20, %zmm54545; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}4546; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %ymm84547; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %ymm74548; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]4549; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %ymm34550; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm44551; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]4552; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm17, %zmm64553; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm114554; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm134555; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[8],ymm11[8],ymm13[9],ymm11[9],ymm13[10],ymm11[10],ymm13[11],ymm11[11]4556; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm24557; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm14558; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]4559; AVX512DQ-FCP-NEXT:    vpermt2d %zmm15, %zmm18, %zmm04560; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, %zmm0 {%k1}4561; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15]4562; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]4563; AVX512DQ-FCP-NEXT:    vpermt2d %zmm6, %zmm17, %zmm34564; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm13[4],ymm11[4],ymm13[5],ymm11[5],ymm13[6],ymm11[6],ymm13[7],ymm11[7],ymm13[12],ymm11[12],ymm13[13],ymm11[13],ymm13[14],ymm11[14],ymm13[15],ymm11[15]4565; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]4566; AVX512DQ-FCP-NEXT:    vpermt2d %zmm4, %zmm18, %zmm14567; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k1}4568; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm21, %xmm24569; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm22, %xmm34570; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]4571; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm23, %xmm34572; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm24, %xmm44573; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]4574; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm16, %zmm34575; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm25, %xmm24576; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm26, %xmm44577; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]4578; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm27, %xmm44579; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]4580; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm20, %zmm44581; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}4582; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax4583; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)4584; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 192(%rax)4585; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 128(%rax)4586; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, 320(%rax)4587; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm14, 256(%rax)4588; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm12, 448(%rax)4589; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm10, 384(%rax)4590; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm19, 64(%rax)4591; AVX512DQ-FCP-NEXT:    vzeroupper4592; AVX512DQ-FCP-NEXT:    retq4593;4594; AVX512BW-LABEL: store_i16_stride8_vf32:4595; AVX512BW:       # %bb.0:4596; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax4597; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r104598; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r114599; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm04600; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm14601; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm24602; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm34603; AVX512BW-NEXT:    vmovdqa64 (%r8), %zmm54604; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm64605; AVX512BW-NEXT:    vmovdqa64 (%r11), %zmm74606; AVX512BW-NEXT:    vmovdqa64 (%r10), %zmm84607; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35]4608; AVX512BW-NEXT:    vpermi2w %zmm8, %zmm7, %zmm44609; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0]4610; AVX512BW-NEXT:    vpermi2w %zmm6, %zmm5, %zmm94611; AVX512BW-NEXT:    movw $-30584, %cx # imm = 0x88884612; AVX512BW-NEXT:    kmovd %ecx, %k14613; AVX512BW-NEXT:    vmovdqa32 %zmm4, %zmm9 {%k1}4614; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0,0,0]4615; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm104616; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0,0,0,0,0]4617; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm44618; AVX512BW-NEXT:    movw $8738, %cx # imm = 0x22224619; AVX512BW-NEXT:    kmovd %ecx, %k24620; AVX512BW-NEXT:    vmovdqa32 %zmm10, %zmm4 {%k2}4621; AVX512BW-NEXT:    movb $-86, %cl4622; AVX512BW-NEXT:    kmovd %ecx, %k34623; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm4 {%k3}4624; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,0,0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39]4625; AVX512BW-NEXT:    vpermi2w %zmm8, %zmm7, %zmm94626; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,0,0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0]4627; AVX512BW-NEXT:    vpermi2w %zmm6, %zmm5, %zmm104628; AVX512BW-NEXT:    vmovdqa32 %zmm9, %zmm10 {%k1}4629; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0,0,0]4630; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm114631; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0,0,0,0,0]4632; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm94633; AVX512BW-NEXT:    vmovdqa32 %zmm11, %zmm9 {%k2}4634; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm9 {%k3}4635; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,0,0,0,0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43]4636; AVX512BW-NEXT:    vpermi2w %zmm8, %zmm7, %zmm104637; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [0,0,0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0]4638; AVX512BW-NEXT:    vpermi2w %zmm6, %zmm5, %zmm114639; AVX512BW-NEXT:    vmovdqa32 %zmm10, %zmm11 {%k1}4640; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0,0,0]4641; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm124642; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0,0,0,0,0]4643; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm104644; AVX512BW-NEXT:    vmovdqa32 %zmm12, %zmm10 {%k2}4645; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm10 {%k3}4646; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [0,0,0,0,0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47]4647; AVX512BW-NEXT:    vpermi2w %zmm8, %zmm7, %zmm114648; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0]4649; AVX512BW-NEXT:    vpermi2w %zmm6, %zmm5, %zmm124650; AVX512BW-NEXT:    vmovdqa32 %zmm11, %zmm12 {%k1}4651; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0,0,0]4652; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm134653; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0,0,0,0,0]4654; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm114655; AVX512BW-NEXT:    vmovdqa32 %zmm13, %zmm11 {%k2}4656; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm11 {%k3}4657; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51]4658; AVX512BW-NEXT:    vpermi2w %zmm8, %zmm7, %zmm124659; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0]4660; AVX512BW-NEXT:    vpermi2w %zmm6, %zmm5, %zmm134661; AVX512BW-NEXT:    vmovdqa32 %zmm12, %zmm13 {%k1}4662; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0,0,0]4663; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm124664; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0,0,0,0,0]4665; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm144666; AVX512BW-NEXT:    vmovdqa32 %zmm12, %zmm14 {%k2}4667; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm14 {%k3}4668; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55]4669; AVX512BW-NEXT:    vpermi2w %zmm8, %zmm7, %zmm124670; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0]4671; AVX512BW-NEXT:    vpermi2w %zmm6, %zmm5, %zmm134672; AVX512BW-NEXT:    vmovdqa32 %zmm12, %zmm13 {%k1}4673; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0,0,0]4674; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm124675; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm15 = [20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0,0,0,0,0]4676; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm154677; AVX512BW-NEXT:    vmovdqa32 %zmm12, %zmm15 {%k2}4678; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm15 {%k3}4679; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59]4680; AVX512BW-NEXT:    vpermi2w %zmm8, %zmm7, %zmm124681; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0]4682; AVX512BW-NEXT:    vpermi2w %zmm6, %zmm5, %zmm134683; AVX512BW-NEXT:    vmovdqa32 %zmm12, %zmm13 {%k1}4684; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0,0,0]4685; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm124686; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm16 = [24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0,0,0,0,0]4687; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm164688; AVX512BW-NEXT:    vmovdqa32 %zmm12, %zmm16 {%k2}4689; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm16 {%k3}4690; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63]4691; AVX512BW-NEXT:    vpermi2w %zmm8, %zmm7, %zmm124692; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [0,0,0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0]4693; AVX512BW-NEXT:    vpermi2w %zmm6, %zmm5, %zmm74694; AVX512BW-NEXT:    vmovdqa32 %zmm12, %zmm7 {%k1}4695; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0,0,0]4696; AVX512BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm54697; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0,0,0,0,0]4698; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm24699; AVX512BW-NEXT:    vmovdqa32 %zmm5, %zmm2 {%k2}4700; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm2 {%k3}4701; AVX512BW-NEXT:    vmovdqa64 %zmm2, 448(%rax)4702; AVX512BW-NEXT:    vmovdqa64 %zmm16, 384(%rax)4703; AVX512BW-NEXT:    vmovdqa64 %zmm15, 320(%rax)4704; AVX512BW-NEXT:    vmovdqa64 %zmm14, 256(%rax)4705; AVX512BW-NEXT:    vmovdqa64 %zmm11, 192(%rax)4706; AVX512BW-NEXT:    vmovdqa64 %zmm10, 128(%rax)4707; AVX512BW-NEXT:    vmovdqa64 %zmm9, 64(%rax)4708; AVX512BW-NEXT:    vmovdqa64 %zmm4, (%rax)4709; AVX512BW-NEXT:    vzeroupper4710; AVX512BW-NEXT:    retq4711;4712; AVX512BW-FCP-LABEL: store_i16_stride8_vf32:4713; AVX512BW-FCP:       # %bb.0:4714; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax4715; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r104716; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r114717; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdi), %zmm04718; AVX512BW-FCP-NEXT:    vmovdqa64 (%rsi), %zmm14719; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdx), %zmm24720; AVX512BW-FCP-NEXT:    vmovdqa64 (%rcx), %zmm34721; AVX512BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm54722; AVX512BW-FCP-NEXT:    vmovdqa64 (%r9), %zmm64723; AVX512BW-FCP-NEXT:    vmovdqa64 (%r11), %zmm74724; AVX512BW-FCP-NEXT:    vmovdqa64 (%r10), %zmm84725; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35]4726; AVX512BW-FCP-NEXT:    vpermi2w %zmm8, %zmm7, %zmm44727; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0]4728; AVX512BW-FCP-NEXT:    vpermi2w %zmm6, %zmm5, %zmm94729; AVX512BW-FCP-NEXT:    movw $-30584, %cx # imm = 0x88884730; AVX512BW-FCP-NEXT:    kmovd %ecx, %k14731; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm4, %zmm9 {%k1}4732; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0,0,0]4733; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm104734; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0,0,0,0,0]4735; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm44736; AVX512BW-FCP-NEXT:    movw $8738, %cx # imm = 0x22224737; AVX512BW-FCP-NEXT:    kmovd %ecx, %k24738; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm10, %zmm4 {%k2}4739; AVX512BW-FCP-NEXT:    movb $-86, %cl4740; AVX512BW-FCP-NEXT:    kmovd %ecx, %k34741; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm9, %zmm4 {%k3}4742; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,0,0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39]4743; AVX512BW-FCP-NEXT:    vpermi2w %zmm8, %zmm7, %zmm94744; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,0,0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0]4745; AVX512BW-FCP-NEXT:    vpermi2w %zmm6, %zmm5, %zmm104746; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm9, %zmm10 {%k1}4747; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0,0,0]4748; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm114749; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0,0,0,0,0]4750; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm94751; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm11, %zmm9 {%k2}4752; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm10, %zmm9 {%k3}4753; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,0,0,0,0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43]4754; AVX512BW-FCP-NEXT:    vpermi2w %zmm8, %zmm7, %zmm104755; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [0,0,0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0]4756; AVX512BW-FCP-NEXT:    vpermi2w %zmm6, %zmm5, %zmm114757; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm10, %zmm11 {%k1}4758; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0,0,0]4759; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm124760; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0,0,0,0,0]4761; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm104762; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm10 {%k2}4763; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm11, %zmm10 {%k3}4764; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [0,0,0,0,0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47]4765; AVX512BW-FCP-NEXT:    vpermi2w %zmm8, %zmm7, %zmm114766; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0]4767; AVX512BW-FCP-NEXT:    vpermi2w %zmm6, %zmm5, %zmm124768; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm11, %zmm12 {%k1}4769; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0,0,0]4770; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm134771; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0,0,0,0,0]4772; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm114773; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm13, %zmm11 {%k2}4774; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm12, %zmm11 {%k3}4775; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51]4776; AVX512BW-FCP-NEXT:    vpermi2w %zmm8, %zmm7, %zmm124777; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0]4778; AVX512BW-FCP-NEXT:    vpermi2w %zmm6, %zmm5, %zmm134779; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm13 {%k1}4780; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0,0,0]4781; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm124782; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0,0,0,0,0]4783; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm144784; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm14 {%k2}4785; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm13, %zmm14 {%k3}4786; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55]4787; AVX512BW-FCP-NEXT:    vpermi2w %zmm8, %zmm7, %zmm124788; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0]4789; AVX512BW-FCP-NEXT:    vpermi2w %zmm6, %zmm5, %zmm134790; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm13 {%k1}4791; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0,0,0]4792; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm124793; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm15 = [20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0,0,0,0,0]4794; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm154795; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm15 {%k2}4796; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm13, %zmm15 {%k3}4797; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59]4798; AVX512BW-FCP-NEXT:    vpermi2w %zmm8, %zmm7, %zmm124799; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0]4800; AVX512BW-FCP-NEXT:    vpermi2w %zmm6, %zmm5, %zmm134801; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm13 {%k1}4802; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0,0,0]4803; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm124804; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm16 = [24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0,0,0,0,0]4805; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm164806; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm16 {%k2}4807; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm13, %zmm16 {%k3}4808; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63]4809; AVX512BW-FCP-NEXT:    vpermi2w %zmm8, %zmm7, %zmm124810; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [0,0,0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0]4811; AVX512BW-FCP-NEXT:    vpermi2w %zmm6, %zmm5, %zmm74812; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm7 {%k1}4813; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0,0,0]4814; AVX512BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm54815; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0,0,0,0,0]4816; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm24817; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm5, %zmm2 {%k2}4818; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm7, %zmm2 {%k3}4819; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm2, 448(%rax)4820; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm16, 384(%rax)4821; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm15, 320(%rax)4822; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm14, 256(%rax)4823; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm11, 192(%rax)4824; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm10, 128(%rax)4825; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm9, 64(%rax)4826; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)4827; AVX512BW-FCP-NEXT:    vzeroupper4828; AVX512BW-FCP-NEXT:    retq4829;4830; AVX512DQ-BW-LABEL: store_i16_stride8_vf32:4831; AVX512DQ-BW:       # %bb.0:4832; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax4833; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %r104834; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %r114835; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdi), %zmm04836; AVX512DQ-BW-NEXT:    vmovdqa64 (%rsi), %zmm14837; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdx), %zmm24838; AVX512DQ-BW-NEXT:    vmovdqa64 (%rcx), %zmm34839; AVX512DQ-BW-NEXT:    vmovdqa64 (%r8), %zmm54840; AVX512DQ-BW-NEXT:    vmovdqa64 (%r9), %zmm64841; AVX512DQ-BW-NEXT:    vmovdqa64 (%r11), %zmm74842; AVX512DQ-BW-NEXT:    vmovdqa64 (%r10), %zmm84843; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35]4844; AVX512DQ-BW-NEXT:    vpermi2w %zmm8, %zmm7, %zmm44845; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0]4846; AVX512DQ-BW-NEXT:    vpermi2w %zmm6, %zmm5, %zmm94847; AVX512DQ-BW-NEXT:    movw $-30584, %cx # imm = 0x88884848; AVX512DQ-BW-NEXT:    kmovd %ecx, %k14849; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm4, %zmm9 {%k1}4850; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0,0,0]4851; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm104852; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0,0,0,0,0]4853; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm44854; AVX512DQ-BW-NEXT:    movw $8738, %cx # imm = 0x22224855; AVX512DQ-BW-NEXT:    kmovd %ecx, %k24856; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm10, %zmm4 {%k2}4857; AVX512DQ-BW-NEXT:    movb $-86, %cl4858; AVX512DQ-BW-NEXT:    kmovd %ecx, %k34859; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm9, %zmm4 {%k3}4860; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,0,0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39]4861; AVX512DQ-BW-NEXT:    vpermi2w %zmm8, %zmm7, %zmm94862; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,0,0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0]4863; AVX512DQ-BW-NEXT:    vpermi2w %zmm6, %zmm5, %zmm104864; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm9, %zmm10 {%k1}4865; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0,0,0]4866; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm114867; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0,0,0,0,0]4868; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm94869; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm11, %zmm9 {%k2}4870; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm10, %zmm9 {%k3}4871; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,0,0,0,0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43]4872; AVX512DQ-BW-NEXT:    vpermi2w %zmm8, %zmm7, %zmm104873; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [0,0,0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0]4874; AVX512DQ-BW-NEXT:    vpermi2w %zmm6, %zmm5, %zmm114875; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm10, %zmm11 {%k1}4876; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0,0,0]4877; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm124878; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0,0,0,0,0]4879; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm104880; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm12, %zmm10 {%k2}4881; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm11, %zmm10 {%k3}4882; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [0,0,0,0,0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47]4883; AVX512DQ-BW-NEXT:    vpermi2w %zmm8, %zmm7, %zmm114884; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0]4885; AVX512DQ-BW-NEXT:    vpermi2w %zmm6, %zmm5, %zmm124886; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm11, %zmm12 {%k1}4887; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0,0,0]4888; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm134889; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0,0,0,0,0]4890; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm114891; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm13, %zmm11 {%k2}4892; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm12, %zmm11 {%k3}4893; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51]4894; AVX512DQ-BW-NEXT:    vpermi2w %zmm8, %zmm7, %zmm124895; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0]4896; AVX512DQ-BW-NEXT:    vpermi2w %zmm6, %zmm5, %zmm134897; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm12, %zmm13 {%k1}4898; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0,0,0]4899; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm124900; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0,0,0,0,0]4901; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm144902; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm12, %zmm14 {%k2}4903; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm13, %zmm14 {%k3}4904; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55]4905; AVX512DQ-BW-NEXT:    vpermi2w %zmm8, %zmm7, %zmm124906; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0]4907; AVX512DQ-BW-NEXT:    vpermi2w %zmm6, %zmm5, %zmm134908; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm12, %zmm13 {%k1}4909; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0,0,0]4910; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm124911; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm15 = [20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0,0,0,0,0]4912; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm154913; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm12, %zmm15 {%k2}4914; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm13, %zmm15 {%k3}4915; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59]4916; AVX512DQ-BW-NEXT:    vpermi2w %zmm8, %zmm7, %zmm124917; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0]4918; AVX512DQ-BW-NEXT:    vpermi2w %zmm6, %zmm5, %zmm134919; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm12, %zmm13 {%k1}4920; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0,0,0]4921; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm124922; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm16 = [24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0,0,0,0,0]4923; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm164924; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm12, %zmm16 {%k2}4925; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm13, %zmm16 {%k3}4926; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63]4927; AVX512DQ-BW-NEXT:    vpermi2w %zmm8, %zmm7, %zmm124928; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [0,0,0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0]4929; AVX512DQ-BW-NEXT:    vpermi2w %zmm6, %zmm5, %zmm74930; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm12, %zmm7 {%k1}4931; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0,0,0]4932; AVX512DQ-BW-NEXT:    vpermi2w %zmm3, %zmm2, %zmm54933; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0,0,0,0,0]4934; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm24935; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm5, %zmm2 {%k2}4936; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm7, %zmm2 {%k3}4937; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm2, 448(%rax)4938; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm16, 384(%rax)4939; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm15, 320(%rax)4940; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm14, 256(%rax)4941; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm11, 192(%rax)4942; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm10, 128(%rax)4943; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm9, 64(%rax)4944; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, (%rax)4945; AVX512DQ-BW-NEXT:    vzeroupper4946; AVX512DQ-BW-NEXT:    retq4947;4948; AVX512DQ-BW-FCP-LABEL: store_i16_stride8_vf32:4949; AVX512DQ-BW-FCP:       # %bb.0:4950; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax4951; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r104952; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r114953; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdi), %zmm04954; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rsi), %zmm14955; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdx), %zmm24956; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rcx), %zmm34957; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm54958; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r9), %zmm64959; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r11), %zmm74960; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r10), %zmm84961; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,0,0,0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35]4962; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm8, %zmm7, %zmm44963; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0]4964; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm6, %zmm5, %zmm94965; AVX512DQ-BW-FCP-NEXT:    movw $-30584, %cx # imm = 0x88884966; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k14967; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm4, %zmm9 {%k1}4968; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0,0,0]4969; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm104970; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0,0,0,0,0]4971; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm44972; AVX512DQ-BW-FCP-NEXT:    movw $8738, %cx # imm = 0x22224973; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k24974; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm10, %zmm4 {%k2}4975; AVX512DQ-BW-FCP-NEXT:    movb $-86, %cl4976; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k34977; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm9, %zmm4 {%k3}4978; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,0,0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39]4979; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm8, %zmm7, %zmm94980; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,0,0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0]4981; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm6, %zmm5, %zmm104982; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm9, %zmm10 {%k1}4983; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0,0,0]4984; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm114985; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0,0,0,0,0]4986; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm94987; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm11, %zmm9 {%k2}4988; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm10, %zmm9 {%k3}4989; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,0,0,0,0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43]4990; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm8, %zmm7, %zmm104991; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [0,0,0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0]4992; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm6, %zmm5, %zmm114993; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm10, %zmm11 {%k1}4994; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0,0,0]4995; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm124996; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0,0,0,0,0]4997; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm104998; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm10 {%k2}4999; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm11, %zmm10 {%k3}5000; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [0,0,0,0,0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47]5001; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm8, %zmm7, %zmm115002; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0]5003; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm6, %zmm5, %zmm125004; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm11, %zmm12 {%k1}5005; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0,0,0]5006; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm135007; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0,0,0,0,0]5008; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm115009; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm13, %zmm11 {%k2}5010; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm12, %zmm11 {%k3}5011; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51]5012; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm8, %zmm7, %zmm125013; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0]5014; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm6, %zmm5, %zmm135015; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm13 {%k1}5016; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0,0,0]5017; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm125018; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0,0,0,0,0]5019; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm145020; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm14 {%k2}5021; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm13, %zmm14 {%k3}5022; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55]5023; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm8, %zmm7, %zmm125024; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0]5025; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm6, %zmm5, %zmm135026; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm13 {%k1}5027; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0,0,0]5028; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm125029; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm15 = [20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0,0,0,0,0]5030; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm155031; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm15 {%k2}5032; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm13, %zmm15 {%k3}5033; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59]5034; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm8, %zmm7, %zmm125035; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0]5036; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm6, %zmm5, %zmm135037; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm13 {%k1}5038; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0,0,0]5039; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm125040; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm16 = [24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0,0,0,0,0]5041; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm165042; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm16 {%k2}5043; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm13, %zmm16 {%k3}5044; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63]5045; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm8, %zmm7, %zmm125046; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [0,0,0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0]5047; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm6, %zmm5, %zmm75048; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm12, %zmm7 {%k1}5049; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0,0,0]5050; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm3, %zmm2, %zmm55051; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0,0,0,0,0]5052; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm0, %zmm25053; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm5, %zmm2 {%k2}5054; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm7, %zmm2 {%k3}5055; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm2, 448(%rax)5056; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm16, 384(%rax)5057; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm15, 320(%rax)5058; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm14, 256(%rax)5059; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm11, 192(%rax)5060; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm10, 128(%rax)5061; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm9, 64(%rax)5062; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, (%rax)5063; AVX512DQ-BW-FCP-NEXT:    vzeroupper5064; AVX512DQ-BW-FCP-NEXT:    retq5065  %in.vec0 = load <32 x i16>, ptr %in.vecptr0, align 645066  %in.vec1 = load <32 x i16>, ptr %in.vecptr1, align 645067  %in.vec2 = load <32 x i16>, ptr %in.vecptr2, align 645068  %in.vec3 = load <32 x i16>, ptr %in.vecptr3, align 645069  %in.vec4 = load <32 x i16>, ptr %in.vecptr4, align 645070  %in.vec5 = load <32 x i16>, ptr %in.vecptr5, align 645071  %in.vec6 = load <32 x i16>, ptr %in.vecptr6, align 645072  %in.vec7 = load <32 x i16>, ptr %in.vecptr7, align 645073  %1 = shufflevector <32 x i16> %in.vec0, <32 x i16> %in.vec1, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>5074  %2 = shufflevector <32 x i16> %in.vec2, <32 x i16> %in.vec3, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>5075  %3 = shufflevector <32 x i16> %in.vec4, <32 x i16> %in.vec5, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>5076  %4 = shufflevector <32 x i16> %in.vec6, <32 x i16> %in.vec7, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>5077  %5 = shufflevector <64 x i16> %1, <64 x i16> %2, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>5078  %6 = shufflevector <64 x i16> %3, <64 x i16> %4, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>5079  %7 = shufflevector <128 x i16> %5, <128 x i16> %6, <256 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127, i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143, i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151, i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159, i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191, i32 192, i32 193, i32 194, i32 195, i32 196, i32 197, i32 198, i32 199, i32 200, i32 201, i32 202, i32 203, i32 204, i32 205, i32 206, i32 207, i32 208, i32 209, i32 210, i32 211, i32 212, i32 213, i32 214, i32 215, i32 216, i32 217, i32 218, i32 219, i32 220, i32 221, i32 222, i32 223, i32 224, i32 225, i32 226, i32 227, i32 228, i32 229, i32 230, i32 231, i32 232, i32 233, i32 234, i32 235, i32 236, i32 237, i32 238, i32 239, i32 240, i32 241, i32 242, i32 243, i32 244, i32 245, i32 246, i32 247, i32 248, i32 249, i32 250, i32 251, i32 252, i32 253, i32 254, i32 255>5080  %interleaved.vec = shufflevector <256 x i16> %7, <256 x i16> poison, <256 x i32> <i32 0, i32 32, i32 64, i32 96, i32 128, i32 160, i32 192, i32 224, i32 1, i32 33, i32 65, i32 97, i32 129, i32 161, i32 193, i32 225, i32 2, i32 34, i32 66, i32 98, i32 130, i32 162, i32 194, i32 226, i32 3, i32 35, i32 67, i32 99, i32 131, i32 163, i32 195, i32 227, i32 4, i32 36, i32 68, i32 100, i32 132, i32 164, i32 196, i32 228, i32 5, i32 37, i32 69, i32 101, i32 133, i32 165, i32 197, i32 229, i32 6, i32 38, i32 70, i32 102, i32 134, i32 166, i32 198, i32 230, i32 7, i32 39, i32 71, i32 103, i32 135, i32 167, i32 199, i32 231, i32 8, i32 40, i32 72, i32 104, i32 136, i32 168, i32 200, i32 232, i32 9, i32 41, i32 73, i32 105, i32 137, i32 169, i32 201, i32 233, i32 10, i32 42, i32 74, i32 106, i32 138, i32 170, i32 202, i32 234, i32 11, i32 43, i32 75, i32 107, i32 139, i32 171, i32 203, i32 235, i32 12, i32 44, i32 76, i32 108, i32 140, i32 172, i32 204, i32 236, i32 13, i32 45, i32 77, i32 109, i32 141, i32 173, i32 205, i32 237, i32 14, i32 46, i32 78, i32 110, i32 142, i32 174, i32 206, i32 238, i32 15, i32 47, i32 79, i32 111, i32 143, i32 175, i32 207, i32 239, i32 16, i32 48, i32 80, i32 112, i32 144, i32 176, i32 208, i32 240, i32 17, i32 49, i32 81, i32 113, i32 145, i32 177, i32 209, i32 241, i32 18, i32 50, i32 82, i32 114, i32 146, i32 178, i32 210, i32 242, i32 19, i32 51, i32 83, i32 115, i32 147, i32 179, i32 211, i32 243, i32 20, i32 52, i32 84, i32 116, i32 148, i32 180, i32 212, i32 244, i32 21, i32 53, i32 85, i32 117, i32 149, i32 181, i32 213, i32 245, i32 22, i32 54, i32 86, i32 118, i32 150, i32 182, i32 214, i32 246, i32 23, i32 55, i32 87, i32 119, i32 151, i32 183, i32 215, i32 247, i32 24, i32 56, i32 88, i32 120, i32 152, i32 184, i32 216, i32 248, i32 25, i32 57, i32 89, i32 121, i32 153, i32 185, i32 217, i32 249, i32 26, i32 58, i32 90, i32 122, i32 154, i32 186, i32 218, i32 250, i32 27, i32 59, i32 91, i32 123, i32 155, i32 187, i32 219, i32 251, i32 28, i32 60, i32 92, i32 124, i32 156, i32 188, i32 220, i32 252, i32 29, i32 61, i32 93, i32 125, i32 157, i32 189, i32 221, i32 253, i32 30, i32 62, i32 94, i32 126, i32 158, i32 190, i32 222, i32 254, i32 31, i32 63, i32 95, i32 127, i32 159, i32 191, i32 223, i32 255>5081  store <256 x i16> %interleaved.vec, ptr %out.vec, align 645082  ret void5083}5084 5085define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecptr2, ptr %in.vecptr3, ptr %in.vecptr4, ptr %in.vecptr5, ptr %in.vecptr6, ptr %in.vecptr7, ptr %out.vec) nounwind {5086; SSE-LABEL: store_i16_stride8_vf64:5087; SSE:       # %bb.0:5088; SSE-NEXT:    subq $776, %rsp # imm = 0x3085089; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %rax5090; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %r105091; SSE-NEXT:    movdqa (%rdi), %xmm25092; SSE-NEXT:    movdqa (%rsi), %xmm75093; SSE-NEXT:    movdqa (%rdx), %xmm35094; SSE-NEXT:    movdqa (%rcx), %xmm95095; SSE-NEXT:    movdqa (%r8), %xmm45096; SSE-NEXT:    movdqa (%r9), %xmm105097; SSE-NEXT:    movdqa (%r10), %xmm85098; SSE-NEXT:    movdqa (%rax), %xmm115099; SSE-NEXT:    movdqa %xmm3, %xmm15100; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm9[0],xmm1[1],xmm9[1],xmm1[2],xmm9[2],xmm1[3],xmm9[3]5101; SSE-NEXT:    movdqa %xmm2, %xmm05102; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1],xmm0[2],xmm7[2],xmm0[3],xmm7[3]5103; SSE-NEXT:    movdqa %xmm0, %xmm55104; SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]5105; SSE-NEXT:    movdqa %xmm8, %xmm125106; SSE-NEXT:    punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]5107; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,0,0]5108; SSE-NEXT:    movdqa %xmm4, %xmm65109; SSE-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm10[0],xmm6[1],xmm10[1],xmm6[2],xmm10[2],xmm6[3],xmm10[3]5110; SSE-NEXT:    pshufd {{.*#+}} xmm14 = xmm6[0,0,0,0]5111; SSE-NEXT:    punpckhdq {{.*#+}} xmm14 = xmm14[2],xmm13[2],xmm14[3],xmm13[3]5112; SSE-NEXT:    movsd {{.*#+}} xmm14 = xmm5[0],xmm14[1]5113; SSE-NEXT:    movapd %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5114; SSE-NEXT:    movdqa %xmm6, %xmm55115; SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm12[0],xmm5[1],xmm12[1]5116; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm1[1,1,1,1]5117; SSE-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[1,1,1,1]5118; SSE-NEXT:    punpckldq {{.*#+}} xmm14 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]5119; SSE-NEXT:    shufps {{.*#+}} xmm14 = xmm14[0,1],xmm5[2,3]5120; SSE-NEXT:    movaps %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5121; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm12[2,2,2,2]5122; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[2,2,2,2]5123; SSE-NEXT:    punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm5[2],xmm13[3],xmm5[3]5124; SSE-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[3,3,3,3]5125; SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]5126; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm0[0],xmm13[1]5127; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5128; SSE-NEXT:    movdqa 16(%r8), %xmm05129; SSE-NEXT:    punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm12[2],xmm6[3],xmm12[3]5130; SSE-NEXT:    movdqa 16(%r9), %xmm55131; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]5132; SSE-NEXT:    punpckldq {{.*#+}} xmm14 = xmm14[0],xmm1[0],xmm14[1],xmm1[1]5133; SSE-NEXT:    movdqa 16(%r10), %xmm15134; SSE-NEXT:    shufps {{.*#+}} xmm14 = xmm14[0,1],xmm6[2,3]5135; SSE-NEXT:    movaps %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5136; SSE-NEXT:    movdqa 16(%rax), %xmm65137; SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm9[4],xmm3[5],xmm9[5],xmm3[6],xmm9[6],xmm3[7],xmm9[7]5138; SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm7[4],xmm2[5],xmm7[5],xmm2[6],xmm7[6],xmm2[7],xmm7[7]5139; SSE-NEXT:    movdqa %xmm2, %xmm75140; SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1]5141; SSE-NEXT:    punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm11[4],xmm8[5],xmm11[5],xmm8[6],xmm11[6],xmm8[7],xmm11[7]5142; SSE-NEXT:    pshufd {{.*#+}} xmm9 = xmm8[0,0,0,0]5143; SSE-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm10[4],xmm4[5],xmm10[5],xmm4[6],xmm10[6],xmm4[7],xmm10[7]5144; SSE-NEXT:    pshufd {{.*#+}} xmm10 = xmm4[0,0,0,0]5145; SSE-NEXT:    punpckhdq {{.*#+}} xmm10 = xmm10[2],xmm9[2],xmm10[3],xmm9[3]5146; SSE-NEXT:    movsd {{.*#+}} xmm10 = xmm7[0],xmm10[1]5147; SSE-NEXT:    movapd %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5148; SSE-NEXT:    movdqa %xmm4, %xmm75149; SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]5150; SSE-NEXT:    pshufd {{.*#+}} xmm9 = xmm3[1,1,1,1]5151; SSE-NEXT:    pshufd {{.*#+}} xmm10 = xmm2[1,1,1,1]5152; SSE-NEXT:    punpckldq {{.*#+}} xmm10 = xmm10[0],xmm9[0],xmm10[1],xmm9[1]5153; SSE-NEXT:    shufps {{.*#+}} xmm10 = xmm10[0,1],xmm7[2,3]5154; SSE-NEXT:    movaps %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5155; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm8[2,2,2,2]5156; SSE-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[2,2,2,2]5157; SSE-NEXT:    punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm7[2],xmm9[3],xmm7[3]5158; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[3,3,3,3]5159; SSE-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]5160; SSE-NEXT:    movsd {{.*#+}} xmm9 = xmm2[0],xmm9[1]5161; SSE-NEXT:    movapd %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5162; SSE-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm8[2],xmm4[3],xmm8[3]5163; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[3,3,3,3]5164; SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]5165; SSE-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,1],xmm4[2,3]5166; SSE-NEXT:    movaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5167; SSE-NEXT:    movdqa %xmm1, %xmm75168; SSE-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]5169; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm7[0,0,0,0]5170; SSE-NEXT:    movdqa %xmm0, %xmm45171; SSE-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]5172; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,0,0,0]5173; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]5174; SSE-NEXT:    movdqa %xmm3, %xmm135175; SSE-NEXT:    movdqa 16(%rdx), %xmm25176; SSE-NEXT:    movdqa 16(%rcx), %xmm85177; SSE-NEXT:    movdqa %xmm2, %xmm105178; SSE-NEXT:    punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]5179; SSE-NEXT:    movdqa 16(%rdi), %xmm35180; SSE-NEXT:    movdqa 16(%rsi), %xmm95181; SSE-NEXT:    movdqa %xmm3, %xmm115182; SSE-NEXT:    punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]5183; SSE-NEXT:    movdqa %xmm11, %xmm125184; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm10[0],xmm12[1],xmm10[1]5185; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm12[0],xmm13[1]5186; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5187; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm10[1,1,1,1]5188; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm11[1,1,1,1]5189; SSE-NEXT:    punpckldq {{.*#+}} xmm13 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]5190; SSE-NEXT:    movdqa %xmm4, %xmm125191; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm7[0],xmm12[1],xmm7[1]5192; SSE-NEXT:    shufps {{.*#+}} xmm13 = xmm13[0,1],xmm12[2,3]5193; SSE-NEXT:    movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5194; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm7[2,2,2,2]5195; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[2,2,2,2]5196; SSE-NEXT:    punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm12[2],xmm13[3],xmm12[3]5197; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[3,3,3,3]5198; SSE-NEXT:    punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm10[2],xmm11[3],xmm10[3]5199; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm11[0],xmm13[1]5200; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5201; SSE-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm7[2],xmm4[3],xmm7[3]5202; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm10[3,3,3,3]5203; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm7[0],xmm12[1],xmm7[1]5204; SSE-NEXT:    shufps {{.*#+}} xmm12 = xmm12[0,1],xmm4[2,3]5205; SSE-NEXT:    movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5206; SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]5207; SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm9[4],xmm3[5],xmm9[5],xmm3[6],xmm9[6],xmm3[7],xmm9[7]5208; SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]5209; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7]5210; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,0,0]5211; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,0,0]5212; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]5213; SSE-NEXT:    movdqa %xmm3, %xmm45214; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]5215; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm4[0],xmm5[1]5216; SSE-NEXT:    movapd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5217; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]5218; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]5219; SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]5220; SSE-NEXT:    movdqa %xmm0, %xmm45221; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]5222; SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm4[2,3]5223; SSE-NEXT:    movaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5224; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,2,2,2]5225; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[2,2,2,2]5226; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]5227; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[3,3,3,3]5228; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]5229; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm3[0],xmm5[1]5230; SSE-NEXT:    movapd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5231; SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]5232; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]5233; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]5234; SSE-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,1],xmm0[2,3]5235; SSE-NEXT:    movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5236; SSE-NEXT:    movdqa 32(%r10), %xmm05237; SSE-NEXT:    movdqa 32(%rax), %xmm45238; SSE-NEXT:    movdqa %xmm0, %xmm65239; SSE-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]5240; SSE-NEXT:    movdqa 32(%r8), %xmm15241; SSE-NEXT:    movdqa 32(%r9), %xmm55242; SSE-NEXT:    movdqa %xmm1, %xmm75243; SSE-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]5244; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[0,0,0,0]5245; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[0,0,0,0]5246; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]5247; SSE-NEXT:    movdqa %xmm3, %xmm135248; SSE-NEXT:    movdqa 32(%rdx), %xmm25249; SSE-NEXT:    movdqa 32(%rcx), %xmm85250; SSE-NEXT:    movdqa %xmm2, %xmm105251; SSE-NEXT:    punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]5252; SSE-NEXT:    movdqa 32(%rdi), %xmm35253; SSE-NEXT:    movdqa 32(%rsi), %xmm95254; SSE-NEXT:    movdqa %xmm3, %xmm115255; SSE-NEXT:    punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]5256; SSE-NEXT:    movdqa %xmm11, %xmm125257; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm10[0],xmm12[1],xmm10[1]5258; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm12[0],xmm13[1]5259; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5260; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm10[1,1,1,1]5261; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm11[1,1,1,1]5262; SSE-NEXT:    punpckldq {{.*#+}} xmm13 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]5263; SSE-NEXT:    movdqa %xmm7, %xmm125264; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm6[0],xmm12[1],xmm6[1]5265; SSE-NEXT:    shufps {{.*#+}} xmm13 = xmm13[0,1],xmm12[2,3]5266; SSE-NEXT:    movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5267; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[2,2,2,2]5268; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[2,2,2,2]5269; SSE-NEXT:    punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm12[2],xmm13[3],xmm12[3]5270; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[3,3,3,3]5271; SSE-NEXT:    punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm10[2],xmm11[3],xmm10[3]5272; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm11[0],xmm13[1]5273; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5274; SSE-NEXT:    punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm6[2],xmm7[3],xmm6[3]5275; SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm10[3,3,3,3]5276; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm6[0],xmm12[1],xmm6[1]5277; SSE-NEXT:    shufps {{.*#+}} xmm12 = xmm12[0,1],xmm7[2,3]5278; SSE-NEXT:    movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5279; SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]5280; SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm9[4],xmm3[5],xmm9[5],xmm3[6],xmm9[6],xmm3[7],xmm9[7]5281; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]5282; SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]5283; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,0,0]5284; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,0,0]5285; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]5286; SSE-NEXT:    movdqa %xmm3, %xmm45287; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]5288; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm4[0],xmm5[1]5289; SSE-NEXT:    movapd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5290; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]5291; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]5292; SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]5293; SSE-NEXT:    movdqa %xmm1, %xmm45294; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]5295; SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm4[2,3]5296; SSE-NEXT:    movaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5297; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,2,2,2]5298; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,2,2,2]5299; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]5300; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[3,3,3,3]5301; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]5302; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm3[0],xmm5[1]5303; SSE-NEXT:    movapd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5304; SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]5305; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]5306; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]5307; SSE-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,1],xmm1[2,3]5308; SSE-NEXT:    movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5309; SSE-NEXT:    movdqa 48(%r10), %xmm05310; SSE-NEXT:    movdqa 48(%rax), %xmm45311; SSE-NEXT:    movdqa %xmm0, %xmm65312; SSE-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]5313; SSE-NEXT:    movdqa 48(%r8), %xmm15314; SSE-NEXT:    movdqa 48(%r9), %xmm55315; SSE-NEXT:    movdqa %xmm1, %xmm75316; SSE-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]5317; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[0,0,0,0]5318; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[0,0,0,0]5319; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]5320; SSE-NEXT:    movdqa %xmm3, %xmm135321; SSE-NEXT:    movdqa 48(%rdx), %xmm25322; SSE-NEXT:    movdqa 48(%rcx), %xmm85323; SSE-NEXT:    movdqa %xmm2, %xmm105324; SSE-NEXT:    punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]5325; SSE-NEXT:    movdqa 48(%rdi), %xmm35326; SSE-NEXT:    movdqa 48(%rsi), %xmm95327; SSE-NEXT:    movdqa %xmm3, %xmm115328; SSE-NEXT:    punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]5329; SSE-NEXT:    movdqa %xmm11, %xmm125330; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm10[0],xmm12[1],xmm10[1]5331; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm12[0],xmm13[1]5332; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5333; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm10[1,1,1,1]5334; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm11[1,1,1,1]5335; SSE-NEXT:    punpckldq {{.*#+}} xmm13 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]5336; SSE-NEXT:    movdqa %xmm7, %xmm125337; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm6[0],xmm12[1],xmm6[1]5338; SSE-NEXT:    shufps {{.*#+}} xmm13 = xmm13[0,1],xmm12[2,3]5339; SSE-NEXT:    movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5340; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[2,2,2,2]5341; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[2,2,2,2]5342; SSE-NEXT:    punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm12[2],xmm13[3],xmm12[3]5343; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[3,3,3,3]5344; SSE-NEXT:    punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm10[2],xmm11[3],xmm10[3]5345; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm11[0],xmm13[1]5346; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5347; SSE-NEXT:    punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm6[2],xmm7[3],xmm6[3]5348; SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm10[3,3,3,3]5349; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm6[0],xmm12[1],xmm6[1]5350; SSE-NEXT:    shufps {{.*#+}} xmm12 = xmm12[0,1],xmm7[2,3]5351; SSE-NEXT:    movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5352; SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]5353; SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm9[4],xmm3[5],xmm9[5],xmm3[6],xmm9[6],xmm3[7],xmm9[7]5354; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]5355; SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]5356; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,0,0]5357; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,0,0]5358; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]5359; SSE-NEXT:    movdqa %xmm3, %xmm45360; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]5361; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm4[0],xmm5[1]5362; SSE-NEXT:    movapd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5363; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]5364; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]5365; SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]5366; SSE-NEXT:    movdqa %xmm1, %xmm45367; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]5368; SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm4[2,3]5369; SSE-NEXT:    movaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5370; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,2,2,2]5371; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,2,2,2]5372; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]5373; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[3,3,3,3]5374; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]5375; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm3[0],xmm5[1]5376; SSE-NEXT:    movapd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5377; SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]5378; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]5379; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]5380; SSE-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,1],xmm1[2,3]5381; SSE-NEXT:    movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5382; SSE-NEXT:    movdqa 64(%r10), %xmm05383; SSE-NEXT:    movdqa 64(%rax), %xmm45384; SSE-NEXT:    movdqa %xmm0, %xmm65385; SSE-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]5386; SSE-NEXT:    movdqa 64(%r8), %xmm15387; SSE-NEXT:    movdqa 64(%r9), %xmm55388; SSE-NEXT:    movdqa %xmm1, %xmm75389; SSE-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]5390; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[0,0,0,0]5391; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[0,0,0,0]5392; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]5393; SSE-NEXT:    movdqa %xmm3, %xmm135394; SSE-NEXT:    movdqa 64(%rdx), %xmm25395; SSE-NEXT:    movdqa 64(%rcx), %xmm85396; SSE-NEXT:    movdqa %xmm2, %xmm105397; SSE-NEXT:    punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]5398; SSE-NEXT:    movdqa 64(%rdi), %xmm35399; SSE-NEXT:    movdqa 64(%rsi), %xmm95400; SSE-NEXT:    movdqa %xmm3, %xmm115401; SSE-NEXT:    punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]5402; SSE-NEXT:    movdqa %xmm11, %xmm125403; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm10[0],xmm12[1],xmm10[1]5404; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm12[0],xmm13[1]5405; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5406; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm10[1,1,1,1]5407; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm11[1,1,1,1]5408; SSE-NEXT:    punpckldq {{.*#+}} xmm13 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]5409; SSE-NEXT:    movdqa %xmm7, %xmm125410; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm6[0],xmm12[1],xmm6[1]5411; SSE-NEXT:    shufps {{.*#+}} xmm13 = xmm13[0,1],xmm12[2,3]5412; SSE-NEXT:    movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5413; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[2,2,2,2]5414; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[2,2,2,2]5415; SSE-NEXT:    punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm12[2],xmm13[3],xmm12[3]5416; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[3,3,3,3]5417; SSE-NEXT:    punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm10[2],xmm11[3],xmm10[3]5418; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm11[0],xmm13[1]5419; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5420; SSE-NEXT:    punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm6[2],xmm7[3],xmm6[3]5421; SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm10[3,3,3,3]5422; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm6[0],xmm12[1],xmm6[1]5423; SSE-NEXT:    shufps {{.*#+}} xmm12 = xmm12[0,1],xmm7[2,3]5424; SSE-NEXT:    movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5425; SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]5426; SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm9[4],xmm3[5],xmm9[5],xmm3[6],xmm9[6],xmm3[7],xmm9[7]5427; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]5428; SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]5429; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,0,0]5430; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,0,0]5431; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]5432; SSE-NEXT:    movdqa %xmm3, %xmm45433; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]5434; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm4[0],xmm5[1]5435; SSE-NEXT:    movapd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5436; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]5437; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]5438; SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]5439; SSE-NEXT:    movdqa %xmm1, %xmm45440; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]5441; SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm4[2,3]5442; SSE-NEXT:    movaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5443; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,2,2,2]5444; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,2,2,2]5445; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]5446; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[3,3,3,3]5447; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]5448; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm3[0],xmm5[1]5449; SSE-NEXT:    movapd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5450; SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]5451; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]5452; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]5453; SSE-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,1],xmm1[2,3]5454; SSE-NEXT:    movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5455; SSE-NEXT:    movdqa 80(%r10), %xmm05456; SSE-NEXT:    movdqa 80(%rax), %xmm45457; SSE-NEXT:    movdqa %xmm0, %xmm65458; SSE-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]5459; SSE-NEXT:    movdqa 80(%r8), %xmm15460; SSE-NEXT:    movdqa 80(%r9), %xmm55461; SSE-NEXT:    movdqa %xmm1, %xmm75462; SSE-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]5463; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[0,0,0,0]5464; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[0,0,0,0]5465; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]5466; SSE-NEXT:    movdqa %xmm3, %xmm135467; SSE-NEXT:    movdqa 80(%rdx), %xmm25468; SSE-NEXT:    movdqa 80(%rcx), %xmm85469; SSE-NEXT:    movdqa %xmm2, %xmm105470; SSE-NEXT:    punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]5471; SSE-NEXT:    movdqa 80(%rdi), %xmm35472; SSE-NEXT:    movdqa 80(%rsi), %xmm95473; SSE-NEXT:    movdqa %xmm3, %xmm115474; SSE-NEXT:    punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]5475; SSE-NEXT:    movdqa %xmm11, %xmm125476; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm10[0],xmm12[1],xmm10[1]5477; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm12[0],xmm13[1]5478; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5479; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm10[1,1,1,1]5480; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm11[1,1,1,1]5481; SSE-NEXT:    punpckldq {{.*#+}} xmm13 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]5482; SSE-NEXT:    movdqa %xmm7, %xmm125483; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm6[0],xmm12[1],xmm6[1]5484; SSE-NEXT:    shufps {{.*#+}} xmm13 = xmm13[0,1],xmm12[2,3]5485; SSE-NEXT:    movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5486; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[2,2,2,2]5487; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[2,2,2,2]5488; SSE-NEXT:    punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm12[2],xmm13[3],xmm12[3]5489; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[3,3,3,3]5490; SSE-NEXT:    punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm10[2],xmm11[3],xmm10[3]5491; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm11[0],xmm13[1]5492; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5493; SSE-NEXT:    punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm6[2],xmm7[3],xmm6[3]5494; SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm10[3,3,3,3]5495; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm6[0],xmm12[1],xmm6[1]5496; SSE-NEXT:    shufps {{.*#+}} xmm12 = xmm12[0,1],xmm7[2,3]5497; SSE-NEXT:    movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5498; SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]5499; SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm9[4],xmm3[5],xmm9[5],xmm3[6],xmm9[6],xmm3[7],xmm9[7]5500; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]5501; SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]5502; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,0,0]5503; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,0,0]5504; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]5505; SSE-NEXT:    movdqa %xmm3, %xmm45506; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]5507; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm4[0],xmm5[1]5508; SSE-NEXT:    movapd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5509; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]5510; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]5511; SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]5512; SSE-NEXT:    movdqa %xmm1, %xmm45513; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]5514; SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm4[2,3]5515; SSE-NEXT:    movaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5516; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,2,2,2]5517; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,2,2,2]5518; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]5519; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[3,3,3,3]5520; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]5521; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm3[0],xmm5[1]5522; SSE-NEXT:    movapd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5523; SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]5524; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]5525; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]5526; SSE-NEXT:    shufps {{.*#+}} xmm4 = xmm4[0,1],xmm1[2,3]5527; SSE-NEXT:    movaps %xmm4, (%rsp) # 16-byte Spill5528; SSE-NEXT:    movdqa 96(%r10), %xmm05529; SSE-NEXT:    movdqa 96(%rax), %xmm45530; SSE-NEXT:    movdqa %xmm0, %xmm65531; SSE-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]5532; SSE-NEXT:    movdqa 96(%r8), %xmm15533; SSE-NEXT:    movdqa 96(%r9), %xmm55534; SSE-NEXT:    movdqa %xmm1, %xmm75535; SSE-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]5536; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[0,0,0,0]5537; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm7[0,0,0,0]5538; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]5539; SSE-NEXT:    movdqa %xmm3, %xmm135540; SSE-NEXT:    movdqa 96(%rdx), %xmm25541; SSE-NEXT:    movdqa 96(%rcx), %xmm85542; SSE-NEXT:    movdqa %xmm2, %xmm105543; SSE-NEXT:    punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]5544; SSE-NEXT:    movdqa 96(%rdi), %xmm35545; SSE-NEXT:    movdqa 96(%rsi), %xmm95546; SSE-NEXT:    movdqa %xmm3, %xmm115547; SSE-NEXT:    punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]5548; SSE-NEXT:    movdqa %xmm11, %xmm125549; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm10[0],xmm12[1],xmm10[1]5550; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm12[0],xmm13[1]5551; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5552; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm10[1,1,1,1]5553; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm11[1,1,1,1]5554; SSE-NEXT:    punpckldq {{.*#+}} xmm13 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]5555; SSE-NEXT:    movdqa %xmm7, %xmm125556; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm6[0],xmm12[1],xmm6[1]5557; SSE-NEXT:    shufps {{.*#+}} xmm13 = xmm13[0,1],xmm12[2,3]5558; SSE-NEXT:    movaps %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5559; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm6[2,2,2,2]5560; SSE-NEXT:    pshufd {{.*#+}} xmm13 = xmm7[2,2,2,2]5561; SSE-NEXT:    punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm12[2],xmm13[3],xmm12[3]5562; SSE-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[3,3,3,3]5563; SSE-NEXT:    punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm10[2],xmm11[3],xmm10[3]5564; SSE-NEXT:    movsd {{.*#+}} xmm13 = xmm11[0],xmm13[1]5565; SSE-NEXT:    movapd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5566; SSE-NEXT:    punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm6[2],xmm7[3],xmm6[3]5567; SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm10[3,3,3,3]5568; SSE-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm6[0],xmm12[1],xmm6[1]5569; SSE-NEXT:    shufps {{.*#+}} xmm12 = xmm12[0,1],xmm7[2,3]5570; SSE-NEXT:    movaps %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5571; SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]5572; SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm9[4],xmm3[5],xmm9[5],xmm3[6],xmm9[6],xmm3[7],xmm9[7]5573; SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]5574; SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]5575; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,0,0]5576; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[0,0,0,0]5577; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]5578; SSE-NEXT:    movdqa %xmm3, %xmm45579; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]5580; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm4[0],xmm5[1]5581; SSE-NEXT:    movapd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5582; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]5583; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]5584; SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]5585; SSE-NEXT:    movdqa %xmm1, %xmm45586; SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]5587; SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,1],xmm4[2,3]5588; SSE-NEXT:    movaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5589; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,2,2,2]5590; SSE-NEXT:    pshufd {{.*#+}} xmm15 = xmm1[2,2,2,2]5591; SSE-NEXT:    punpckhdq {{.*#+}} xmm15 = xmm15[2],xmm4[2],xmm15[3],xmm4[3]5592; SSE-NEXT:    pshufd {{.*#+}} xmm14 = xmm3[3,3,3,3]5593; SSE-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]5594; SSE-NEXT:    movsd {{.*#+}} xmm15 = xmm3[0],xmm15[1]5595; SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]5596; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]5597; SSE-NEXT:    punpckldq {{.*#+}} xmm14 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]5598; SSE-NEXT:    shufps {{.*#+}} xmm14 = xmm14[0,1],xmm1[2,3]5599; SSE-NEXT:    movdqa 112(%r10), %xmm95600; SSE-NEXT:    movdqa 112(%rax), %xmm05601; SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5602; SSE-NEXT:    movdqa %xmm9, %xmm55603; SSE-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]5604; SSE-NEXT:    movdqa 112(%r8), %xmm45605; SSE-NEXT:    movdqa 112(%r9), %xmm05606; SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5607; SSE-NEXT:    movdqa %xmm4, %xmm15608; SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]5609; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,0,0]5610; SSE-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[0,0,0,0]5611; SSE-NEXT:    punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm0[2],xmm11[3],xmm0[3]5612; SSE-NEXT:    movdqa 112(%rdx), %xmm65613; SSE-NEXT:    movdqa 112(%rcx), %xmm135614; SSE-NEXT:    movdqa %xmm6, %xmm35615; SSE-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm13[0],xmm3[1],xmm13[1],xmm3[2],xmm13[2],xmm3[3],xmm13[3]5616; SSE-NEXT:    movdqa 112(%rdi), %xmm25617; SSE-NEXT:    movdqa 112(%rsi), %xmm125618; SSE-NEXT:    movdqa %xmm2, %xmm05619; SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm12[0],xmm0[1],xmm12[1],xmm0[2],xmm12[2],xmm0[3],xmm12[3]5620; SSE-NEXT:    movdqa %xmm0, %xmm75621; SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1]5622; SSE-NEXT:    movsd {{.*#+}} xmm11 = xmm7[0],xmm11[1]5623; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,1,1]5624; SSE-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[1,1,1,1]5625; SSE-NEXT:    punpckldq {{.*#+}} xmm10 = xmm10[0],xmm7[0],xmm10[1],xmm7[1]5626; SSE-NEXT:    movdqa %xmm1, %xmm75627; SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]5628; SSE-NEXT:    shufps {{.*#+}} xmm10 = xmm10[0,1],xmm7[2,3]5629; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[2,2,2,2]5630; SSE-NEXT:    pshufd {{.*#+}} xmm8 = xmm1[2,2,2,2]5631; SSE-NEXT:    punpckhdq {{.*#+}} xmm8 = xmm8[2],xmm7[2],xmm8[3],xmm7[3]5632; SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[3,3,3,3]5633; SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]5634; SSE-NEXT:    movsd {{.*#+}} xmm8 = xmm0[0],xmm8[1]5635; SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]5636; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[3,3,3,3]5637; SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1]5638; SSE-NEXT:    shufps {{.*#+}} xmm7 = xmm7[0,1],xmm1[2,3]5639; SSE-NEXT:    punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm13[4],xmm6[5],xmm13[5],xmm6[6],xmm13[6],xmm6[7],xmm13[7]5640; SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm12[4],xmm2[5],xmm12[5],xmm2[6],xmm12[6],xmm2[7],xmm12[7]5641; SSE-NEXT:    punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload5642; SSE-NEXT:    # xmm9 = xmm9[4],mem[4],xmm9[5],mem[5],xmm9[6],mem[6],xmm9[7],mem[7]5643; SSE-NEXT:    punpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload5644; SSE-NEXT:    # xmm4 = xmm4[4],mem[4],xmm4[5],mem[5],xmm4[6],mem[6],xmm4[7],mem[7]5645; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,0,0]5646; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[0,0,0,0]5647; SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]5648; SSE-NEXT:    movdqa %xmm2, %xmm05649; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]5650; SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]5651; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,1,1]5652; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,1,1]5653; SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]5654; SSE-NEXT:    movdqa %xmm4, %xmm05655; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1]5656; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,1],xmm0[2,3]5657; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[2,2,2,2]5658; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[2,2,2,2]5659; SSE-NEXT:    punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm0[2],xmm5[3],xmm0[3]5660; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]5661; SSE-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm6[2],xmm2[3],xmm6[3]5662; SSE-NEXT:    movsd {{.*#+}} xmm5 = xmm2[0],xmm5[1]5663; SSE-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm9[2],xmm4[3],xmm9[3]5664; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[3,3,3,3]5665; SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]5666; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm4[2,3]5667; SSE-NEXT:    movq {{[0-9]+}}(%rsp), %rax5668; SSE-NEXT:    movaps %xmm0, 1008(%rax)5669; SSE-NEXT:    movapd %xmm5, 992(%rax)5670; SSE-NEXT:    movaps %xmm3, 976(%rax)5671; SSE-NEXT:    movapd %xmm1, 960(%rax)5672; SSE-NEXT:    movaps %xmm7, 944(%rax)5673; SSE-NEXT:    movapd %xmm8, 928(%rax)5674; SSE-NEXT:    movaps %xmm10, 912(%rax)5675; SSE-NEXT:    movapd %xmm11, 896(%rax)5676; SSE-NEXT:    movaps %xmm14, 880(%rax)5677; SSE-NEXT:    movapd %xmm15, 864(%rax)5678; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5679; SSE-NEXT:    movaps %xmm0, 848(%rax)5680; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5681; SSE-NEXT:    movaps %xmm0, 832(%rax)5682; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5683; SSE-NEXT:    movaps %xmm0, 816(%rax)5684; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5685; SSE-NEXT:    movaps %xmm0, 800(%rax)5686; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5687; SSE-NEXT:    movaps %xmm0, 784(%rax)5688; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5689; SSE-NEXT:    movaps %xmm0, 768(%rax)5690; SSE-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload5691; SSE-NEXT:    movaps %xmm0, 752(%rax)5692; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5693; SSE-NEXT:    movaps %xmm0, 736(%rax)5694; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5695; SSE-NEXT:    movaps %xmm0, 720(%rax)5696; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5697; SSE-NEXT:    movaps %xmm0, 704(%rax)5698; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5699; SSE-NEXT:    movaps %xmm0, 688(%rax)5700; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5701; SSE-NEXT:    movaps %xmm0, 672(%rax)5702; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5703; SSE-NEXT:    movaps %xmm0, 656(%rax)5704; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5705; SSE-NEXT:    movaps %xmm0, 640(%rax)5706; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5707; SSE-NEXT:    movaps %xmm0, 624(%rax)5708; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5709; SSE-NEXT:    movaps %xmm0, 608(%rax)5710; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5711; SSE-NEXT:    movaps %xmm0, 592(%rax)5712; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5713; SSE-NEXT:    movaps %xmm0, 576(%rax)5714; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5715; SSE-NEXT:    movaps %xmm0, 560(%rax)5716; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5717; SSE-NEXT:    movaps %xmm0, 544(%rax)5718; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5719; SSE-NEXT:    movaps %xmm0, 528(%rax)5720; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5721; SSE-NEXT:    movaps %xmm0, 512(%rax)5722; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5723; SSE-NEXT:    movaps %xmm0, 496(%rax)5724; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5725; SSE-NEXT:    movaps %xmm0, 480(%rax)5726; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5727; SSE-NEXT:    movaps %xmm0, 464(%rax)5728; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5729; SSE-NEXT:    movaps %xmm0, 448(%rax)5730; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5731; SSE-NEXT:    movaps %xmm0, 432(%rax)5732; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5733; SSE-NEXT:    movaps %xmm0, 416(%rax)5734; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5735; SSE-NEXT:    movaps %xmm0, 400(%rax)5736; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5737; SSE-NEXT:    movaps %xmm0, 384(%rax)5738; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5739; SSE-NEXT:    movaps %xmm0, 368(%rax)5740; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5741; SSE-NEXT:    movaps %xmm0, 352(%rax)5742; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5743; SSE-NEXT:    movaps %xmm0, 336(%rax)5744; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5745; SSE-NEXT:    movaps %xmm0, 320(%rax)5746; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5747; SSE-NEXT:    movaps %xmm0, 304(%rax)5748; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5749; SSE-NEXT:    movaps %xmm0, 288(%rax)5750; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5751; SSE-NEXT:    movaps %xmm0, 272(%rax)5752; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5753; SSE-NEXT:    movaps %xmm0, 256(%rax)5754; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5755; SSE-NEXT:    movaps %xmm0, 240(%rax)5756; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5757; SSE-NEXT:    movaps %xmm0, 224(%rax)5758; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5759; SSE-NEXT:    movaps %xmm0, 208(%rax)5760; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5761; SSE-NEXT:    movaps %xmm0, 192(%rax)5762; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5763; SSE-NEXT:    movaps %xmm0, 176(%rax)5764; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5765; SSE-NEXT:    movaps %xmm0, 160(%rax)5766; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5767; SSE-NEXT:    movaps %xmm0, 144(%rax)5768; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5769; SSE-NEXT:    movaps %xmm0, 128(%rax)5770; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5771; SSE-NEXT:    movaps %xmm0, 112(%rax)5772; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5773; SSE-NEXT:    movaps %xmm0, 96(%rax)5774; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5775; SSE-NEXT:    movaps %xmm0, 80(%rax)5776; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5777; SSE-NEXT:    movaps %xmm0, 64(%rax)5778; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5779; SSE-NEXT:    movaps %xmm0, 48(%rax)5780; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5781; SSE-NEXT:    movaps %xmm0, 32(%rax)5782; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5783; SSE-NEXT:    movaps %xmm0, 16(%rax)5784; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5785; SSE-NEXT:    movaps %xmm0, (%rax)5786; SSE-NEXT:    addq $776, %rsp # imm = 0x3085787; SSE-NEXT:    retq5788;5789; AVX-LABEL: store_i16_stride8_vf64:5790; AVX:       # %bb.0:5791; AVX-NEXT:    subq $744, %rsp # imm = 0x2E85792; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax5793; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %r105794; AVX-NEXT:    vmovdqa (%r10), %xmm55795; AVX-NEXT:    vmovdqa 16(%r10), %xmm05796; AVX-NEXT:    vmovdqa (%rax), %xmm65797; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]5798; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[0,0,0,0]5799; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[0,1,0,1]5800; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm15801; AVX-NEXT:    vmovdqa (%r9), %xmm75802; AVX-NEXT:    vmovdqa (%r8), %xmm85803; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]5804; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[0,1,0,1]5805; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm9 = xmm3[0],zero,xmm3[1],zero5806; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm4, %ymm45807; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm4[0,1,2],ymm1[3],ymm4[4,5,6],ymm1[7]5808; AVX-NEXT:    vmovdqa (%rsi), %xmm95809; AVX-NEXT:    vmovdqa (%rdi), %xmm105810; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]5811; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[1,1,1,1]5812; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm15813; AVX-NEXT:    vmovdqa (%rcx), %xmm115814; AVX-NEXT:    vmovdqa (%rdx), %xmm125815; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]5816; AVX-NEXT:    vpshufd {{.*#+}} xmm15 = xmm14[0,0,1,1]5817; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm15, %ymm155818; AVX-NEXT:    vblendps {{.*#+}} ymm15 = ymm1[0],ymm15[1],ymm1[2,3,4],ymm15[5],ymm1[6,7]5819; AVX-NEXT:    vmovdqa 16(%rax), %xmm15820; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm15[0,1],ymm13[2,3],ymm15[4,5],ymm13[6,7]5821; AVX-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5822; AVX-NEXT:    vpshufd {{.*#+}} xmm13 = xmm3[2,2,3,3]5823; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm3, %ymm35824; AVX-NEXT:    vpshufd {{.*#+}} xmm13 = xmm2[2,2,2,2]5825; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm13, %ymm135826; AVX-NEXT:    vmovdqa 16(%r9), %xmm25827; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm3[0,1,2],ymm13[3],ymm3[4,5,6],ymm13[7]5828; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm14[2,2,3,3]5829; AVX-NEXT:    vpshufd {{.*#+}} xmm14 = xmm14[2,3,2,3]5830; AVX-NEXT:    vinsertf128 $1, %xmm14, %ymm3, %ymm35831; AVX-NEXT:    vpshufd {{.*#+}} xmm14 = xmm4[2,3,2,3]5832; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]5833; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm14, %ymm145834; AVX-NEXT:    vmovdqa 16(%r8), %xmm45835; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0],ymm3[1],ymm14[2,3,4],ymm3[5],ymm14[6,7]5836; AVX-NEXT:    vmovdqa 16(%rsi), %xmm35837; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm14[0,1],ymm13[2,3],ymm14[4,5],ymm13[6,7]5838; AVX-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5839; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]5840; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm5[0,0,0,0]5841; AVX-NEXT:    vpshufd {{.*#+}} xmm13 = xmm5[0,1,0,1]5842; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm6, %ymm65843; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]5844; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm8[0,1,0,1]5845; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm13 = xmm8[0],zero,xmm8[1],zero5846; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm7, %ymm135847; AVX-NEXT:    vmovdqa 16(%rdi), %xmm75848; AVX-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2],ymm6[3],ymm13[4,5,6],ymm6[7]5849; AVX-NEXT:    vmovdqa 16(%rcx), %xmm65850; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]5851; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm9[1,1,1,1]5852; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm9, %ymm105853; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]5854; AVX-NEXT:    vpshufd {{.*#+}} xmm12 = xmm11[0,0,1,1]5855; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm12, %ymm125856; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0],ymm12[1],ymm10[2,3,4],ymm12[5],ymm10[6,7]5857; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1],ymm13[2,3],ymm10[4,5],ymm13[6,7]5858; AVX-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5859; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm8[2,2,3,3]5860; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm8, %ymm85861; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm5[2,2,2,2]5862; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm10, %ymm55863; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm8[0,1,2],ymm5[3],ymm8[4,5,6],ymm5[7]5864; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm11[2,2,3,3]5865; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm11[2,3,2,3]5866; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm8, %ymm85867; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm9[2,3,2,3]5868; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[3,3,3,3]5869; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm10, %ymm95870; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0],ymm8[1],ymm9[2,3,4],ymm8[5],ymm9[6,7]5871; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm8[0,1],ymm5[2,3],ymm8[4,5],ymm5[6,7]5872; AVX-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5873; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]5874; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm5[0,0,0,0]5875; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm5[0,1,0,1]5876; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm8, %ymm85877; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]5878; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm9[0,1,0,1]5879; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm11 = xmm9[0],zero,xmm9[1],zero5880; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm10, %ymm105881; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1,2],ymm8[3],ymm10[4,5,6],ymm8[7]5882; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]5883; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm11[1,1,1,1]5884; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm125885; AVX-NEXT:    vmovdqa 16(%rdx), %xmm85886; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]5887; AVX-NEXT:    vpshufd {{.*#+}} xmm14 = xmm13[0,0,1,1]5888; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm14, %ymm145889; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0],ymm14[1],ymm12[2,3,4],ymm14[5],ymm12[6,7]5890; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm12[0,1],ymm10[2,3],ymm12[4,5],ymm10[6,7]5891; AVX-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5892; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm9[2,2,3,3]5893; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm9, %ymm95894; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm5[2,2,2,2]5895; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm10, %ymm55896; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2],ymm5[3],ymm9[4,5,6],ymm5[7]5897; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm13[2,2,3,3]5898; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm13[2,3,2,3]5899; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm5, %ymm55900; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm11[2,3,2,3]5901; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]5902; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm10, %ymm105903; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0],ymm5[1],ymm10[2,3,4],ymm5[5],ymm10[6,7]5904; AVX-NEXT:    vmovdqa 32(%r10), %xmm55905; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0,1],ymm9[2,3],ymm10[4,5],ymm9[6,7]5906; AVX-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5907; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]5908; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm9[0,0,0,0]5909; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm9[0,1,0,1]5910; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm05911; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]5912; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[0,1,0,1]5913; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm4[0],zero,xmm4[1],zero5914; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm25915; AVX-NEXT:    vmovdqa 32(%rax), %xmm15916; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm2[0,1,2],ymm0[3],ymm2[4,5,6],ymm0[7]5917; AVX-NEXT:    vmovdqa 32(%r9), %xmm05918; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]5919; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[1,1,1,1]5920; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm25921; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]5922; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[0,0,1,1]5923; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm7, %ymm75924; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm2[0],ymm7[1],ymm2[2,3,4],ymm7[5],ymm2[6,7]5925; AVX-NEXT:    vmovdqa 32(%r8), %xmm25926; AVX-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1],ymm10[2,3],ymm7[4,5],ymm10[6,7]5927; AVX-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5928; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm4[2,2,3,3]5929; AVX-NEXT:    vinsertf128 $1, %xmm7, %ymm4, %ymm45930; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm9[2,2,2,2]5931; AVX-NEXT:    vinsertf128 $1, %xmm9, %ymm7, %ymm75932; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm7[3],ymm4[4,5,6],ymm7[7]5933; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[2,2,3,3]5934; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,2,3]5935; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm7, %ymm65936; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm3[2,3,2,3]5937; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]5938; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm7, %ymm35939; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm6[1],ymm3[2,3,4],ymm6[5],ymm3[6,7]5940; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]5941; AVX-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5942; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]5943; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm8[0,0,0,0]5944; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm8[0,1,0,1]5945; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm35946; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]5947; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm9[0,1,0,1]5948; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm9[0],zero,xmm9[1],zero5949; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm4, %ymm45950; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm4[0,1,2],ymm3[3],ymm4[4,5,6],ymm3[7]5951; AVX-NEXT:    vmovdqa 32(%rsi), %xmm35952; AVX-NEXT:    vmovdqa 32(%rdi), %xmm45953; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]5954; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm11[1,1,1,1]5955; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm11, %ymm125956; AVX-NEXT:    vmovdqa 32(%rcx), %xmm65957; AVX-NEXT:    vmovdqa 32(%rdx), %xmm75958; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]5959; AVX-NEXT:    vpshufd {{.*#+}} xmm14 = xmm13[0,0,1,1]5960; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm14, %ymm145961; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0],ymm14[1],ymm12[2,3,4],ymm14[5],ymm12[6,7]5962; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm12[0,1],ymm10[2,3],ymm12[4,5],ymm10[6,7]5963; AVX-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5964; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm9[2,2,3,3]5965; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm9, %ymm95966; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm8[2,2,2,2]5967; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm10, %ymm85968; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1,2],ymm8[3],ymm9[4,5,6],ymm8[7]5969; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm13[2,2,3,3]5970; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm13[2,3,2,3]5971; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm9, %ymm95972; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm11[2,3,2,3]5973; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]5974; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm10, %ymm105975; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0],ymm9[1],ymm10[2,3,4],ymm9[5],ymm10[6,7]5976; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1],ymm8[2,3],ymm9[4,5],ymm8[6,7]5977; AVX-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5978; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]5979; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[0,0,0,0]5980; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm1[0,1,0,1]5981; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm5, %ymm55982; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]5983; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[0,1,0,1]5984; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm0[0],zero,xmm0[1],zero5985; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm2, %ymm25986; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm5[3],ymm2[4,5,6],ymm5[7]5987; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]5988; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]5989; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]5990; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm55991; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[0,0,1,1]5992; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm6, %ymm65993; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm6[1],ymm5[2,3,4],ymm6[5],ymm5[6,7]5994; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm5[0,1],ymm2[2,3],ymm5[4,5],ymm2[6,7]5995; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5996; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,2,3,3]5997; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm25998; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[2,2,2,2]5999; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm16000; AVX-NEXT:    vmovdqa 48(%r10), %xmm06001; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]6002; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[2,2,3,3]6003; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]6004; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm46005; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]6006; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]6007; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm36008; AVX-NEXT:    vmovdqa 48(%rax), %xmm16009; AVX-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3,4],ymm4[5],ymm3[6,7]6010; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7]6011; AVX-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6012; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]6013; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm8[0,0,0,0]6014; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm8[0,1,0,1]6015; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm46016; AVX-NEXT:    vmovdqa 48(%r9), %xmm26017; AVX-NEXT:    vmovdqa 48(%r8), %xmm36018; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]6019; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm9[0,1,0,1]6020; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm9[0],zero,xmm9[1],zero6021; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm56022; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm5[0,1,2],ymm4[3],ymm5[4,5,6],ymm4[7]6023; AVX-NEXT:    vmovdqa 48(%rsi), %xmm46024; AVX-NEXT:    vmovdqa 48(%rdi), %xmm56025; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]6026; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm11[1,1,1,1]6027; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm11, %ymm126028; AVX-NEXT:    vmovdqa 48(%rcx), %xmm66029; AVX-NEXT:    vmovdqa 48(%rdx), %xmm76030; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]6031; AVX-NEXT:    vpshufd {{.*#+}} xmm14 = xmm13[0,0,1,1]6032; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm14, %ymm146033; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0],ymm14[1],ymm12[2,3,4],ymm14[5],ymm12[6,7]6034; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm12[0,1],ymm10[2,3],ymm12[4,5],ymm10[6,7]6035; AVX-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6036; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm9[2,2,3,3]6037; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm9, %ymm96038; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm8[2,2,2,2]6039; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm10, %ymm86040; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1,2],ymm8[3],ymm9[4,5,6],ymm8[7]6041; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm13[2,2,3,3]6042; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm13[2,3,2,3]6043; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm9, %ymm96044; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm11[2,3,2,3]6045; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]6046; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm10, %ymm106047; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0],ymm9[1],ymm10[2,3,4],ymm9[5],ymm10[6,7]6048; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1],ymm8[2,3],ymm9[4,5],ymm8[6,7]6049; AVX-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6050; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]6051; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]6052; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm0[0,1,0,1]6053; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm1, %ymm16054; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]6055; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[0,1,0,1]6056; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm2[0],zero,xmm2[1],zero6057; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm3, %ymm36058; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5,6],ymm1[7]6059; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]6060; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]6061; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]6062; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm56063; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[0,0,1,1]6064; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm6, %ymm66065; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm6[1],ymm5[2,3,4],ymm6[5],ymm5[6,7]6066; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm5[0,1],ymm1[2,3],ymm5[4,5],ymm1[6,7]6067; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6068; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]6069; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm16070; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,2,2,2]6071; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm06072; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6],ymm0[7]6073; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[2,2,3,3]6074; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]6075; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm16076; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]6077; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]6078; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm26079; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3,4],ymm1[5],ymm2[6,7]6080; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]6081; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6082; AVX-NEXT:    vmovdqa 64(%r10), %xmm06083; AVX-NEXT:    vmovdqa 64(%rax), %xmm16084; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]6085; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm8[0,0,0,0]6086; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm8[0,1,0,1]6087; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm46088; AVX-NEXT:    vmovdqa 64(%r9), %xmm26089; AVX-NEXT:    vmovdqa 64(%r8), %xmm36090; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]6091; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm9[0,1,0,1]6092; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm9[0],zero,xmm9[1],zero6093; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm56094; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm5[0,1,2],ymm4[3],ymm5[4,5,6],ymm4[7]6095; AVX-NEXT:    vmovdqa 64(%rsi), %xmm46096; AVX-NEXT:    vmovdqa 64(%rdi), %xmm56097; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]6098; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm11[1,1,1,1]6099; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm11, %ymm126100; AVX-NEXT:    vmovdqa 64(%rcx), %xmm66101; AVX-NEXT:    vmovdqa 64(%rdx), %xmm76102; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]6103; AVX-NEXT:    vpshufd {{.*#+}} xmm14 = xmm13[0,0,1,1]6104; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm14, %ymm146105; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0],ymm14[1],ymm12[2,3,4],ymm14[5],ymm12[6,7]6106; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm12[0,1],ymm10[2,3],ymm12[4,5],ymm10[6,7]6107; AVX-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6108; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm9[2,2,3,3]6109; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm9, %ymm96110; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm8[2,2,2,2]6111; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm10, %ymm86112; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1,2],ymm8[3],ymm9[4,5,6],ymm8[7]6113; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm13[2,2,3,3]6114; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm13[2,3,2,3]6115; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm9, %ymm96116; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm11[2,3,2,3]6117; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]6118; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm10, %ymm106119; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0],ymm9[1],ymm10[2,3,4],ymm9[5],ymm10[6,7]6120; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1],ymm8[2,3],ymm9[4,5],ymm8[6,7]6121; AVX-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6122; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]6123; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]6124; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm0[0,1,0,1]6125; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm1, %ymm16126; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]6127; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[0,1,0,1]6128; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm2[0],zero,xmm2[1],zero6129; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm3, %ymm36130; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5,6],ymm1[7]6131; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]6132; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]6133; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]6134; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm56135; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[0,0,1,1]6136; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm6, %ymm66137; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm6[1],ymm5[2,3,4],ymm6[5],ymm5[6,7]6138; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm5[0,1],ymm1[2,3],ymm5[4,5],ymm1[6,7]6139; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6140; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]6141; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm16142; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,2,2,2]6143; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm06144; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6],ymm0[7]6145; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[2,2,3,3]6146; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]6147; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm16148; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]6149; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]6150; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm26151; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3,4],ymm1[5],ymm2[6,7]6152; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]6153; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6154; AVX-NEXT:    vmovdqa 80(%r10), %xmm06155; AVX-NEXT:    vmovdqa 80(%rax), %xmm16156; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]6157; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm8[0,0,0,0]6158; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm8[0,1,0,1]6159; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm46160; AVX-NEXT:    vmovdqa 80(%r9), %xmm26161; AVX-NEXT:    vmovdqa 80(%r8), %xmm36162; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]6163; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm9[0,1,0,1]6164; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm9[0],zero,xmm9[1],zero6165; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm56166; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm5[0,1,2],ymm4[3],ymm5[4,5,6],ymm4[7]6167; AVX-NEXT:    vmovdqa 80(%rsi), %xmm46168; AVX-NEXT:    vmovdqa 80(%rdi), %xmm56169; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]6170; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm11[1,1,1,1]6171; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm11, %ymm126172; AVX-NEXT:    vmovdqa 80(%rcx), %xmm66173; AVX-NEXT:    vmovdqa 80(%rdx), %xmm76174; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]6175; AVX-NEXT:    vpshufd {{.*#+}} xmm14 = xmm13[0,0,1,1]6176; AVX-NEXT:    vinsertf128 $1, %xmm13, %ymm14, %ymm146177; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0],ymm14[1],ymm12[2,3,4],ymm14[5],ymm12[6,7]6178; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm12[0,1],ymm10[2,3],ymm12[4,5],ymm10[6,7]6179; AVX-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6180; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm9[2,2,3,3]6181; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm9, %ymm96182; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm8[2,2,2,2]6183; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm10, %ymm86184; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1,2],ymm8[3],ymm9[4,5,6],ymm8[7]6185; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm13[2,2,3,3]6186; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm13[2,3,2,3]6187; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm9, %ymm96188; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm11[2,3,2,3]6189; AVX-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]6190; AVX-NEXT:    vinsertf128 $1, %xmm11, %ymm10, %ymm106191; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0],ymm9[1],ymm10[2,3,4],ymm9[5],ymm10[6,7]6192; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1],ymm8[2,3],ymm9[4,5],ymm8[6,7]6193; AVX-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6194; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]6195; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]6196; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm0[0,1,0,1]6197; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm1, %ymm16198; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]6199; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[0,1,0,1]6200; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm2[0],zero,xmm2[1],zero6201; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm3, %ymm36202; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5,6],ymm1[7]6203; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]6204; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]6205; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]6206; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm56207; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[0,0,1,1]6208; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm6, %ymm66209; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm6[1],ymm5[2,3,4],ymm6[5],ymm5[6,7]6210; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm5[0,1],ymm1[2,3],ymm5[4,5],ymm1[6,7]6211; AVX-NEXT:    vmovups %ymm1, (%rsp) # 32-byte Spill6212; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]6213; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm16214; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,2,2,2]6215; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm06216; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6],ymm0[7]6217; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[2,2,3,3]6218; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]6219; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm16220; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]6221; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]6222; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm26223; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3,4],ymm1[5],ymm2[6,7]6224; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]6225; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6226; AVX-NEXT:    vmovdqa 96(%r10), %xmm136227; AVX-NEXT:    vmovdqa 96(%rax), %xmm16228; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm1[0],xmm13[0],xmm1[1],xmm13[1],xmm1[2],xmm13[2],xmm1[3],xmm13[3]6229; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm8[0,0,0,0]6230; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm8[0,1,0,1]6231; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm46232; AVX-NEXT:    vmovdqa 96(%r9), %xmm26233; AVX-NEXT:    vmovdqa 96(%r8), %xmm36234; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]6235; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm9[0,1,0,1]6236; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm6 = xmm9[0],zero,xmm9[1],zero6237; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm5, %ymm56238; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm5[0,1,2],ymm4[3],ymm5[4,5,6],ymm4[7]6239; AVX-NEXT:    vmovdqa 96(%rsi), %xmm46240; AVX-NEXT:    vmovdqa 96(%rdi), %xmm56241; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]6242; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm11[1,1,1,1]6243; AVX-NEXT:    vinsertf128 $1, %xmm6, %ymm11, %ymm126244; AVX-NEXT:    vmovdqa 96(%rcx), %xmm66245; AVX-NEXT:    vmovdqa 96(%rdx), %xmm76246; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]6247; AVX-NEXT:    vpshufd {{.*#+}} xmm14 = xmm0[0,0,1,1]6248; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm14, %ymm146249; AVX-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0],ymm14[1],ymm12[2,3,4],ymm14[5],ymm12[6,7]6250; AVX-NEXT:    vblendps {{.*#+}} ymm10 = ymm12[0,1],ymm10[2,3],ymm12[4,5],ymm10[6,7]6251; AVX-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6252; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm9[2,2,3,3]6253; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm9, %ymm96254; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm8[2,2,2,2]6255; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm10, %ymm86256; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1,2],ymm8[3],ymm9[4,5,6],ymm8[7]6257; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm0[2,2,3,3]6258; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]6259; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm9, %ymm06260; AVX-NEXT:    vpshufd {{.*#+}} xmm9 = xmm11[2,3,2,3]6261; AVX-NEXT:    vpshufd {{.*#+}} xmm10 = xmm11[3,3,3,3]6262; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm9, %ymm96263; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm9[0],ymm0[1],ymm9[2,3,4],ymm0[5],ymm9[6,7]6264; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm8[2,3],ymm0[4,5],ymm8[6,7]6265; AVX-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6266; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm13[4],xmm1[5],xmm13[5],xmm1[6],xmm13[6],xmm1[7],xmm13[7]6267; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]6268; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm0[0,1,0,1]6269; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm1, %ymm16270; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]6271; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[0,1,0,1]6272; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm2[0],zero,xmm2[1],zero6273; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm3, %ymm36274; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5,6],ymm1[7]6275; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]6276; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]6277; AVX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]6278; AVX-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm56279; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[0,0,1,1]6280; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm6, %ymm66281; AVX-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm6[1],ymm5[2,3,4],ymm6[5],ymm5[6,7]6282; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm5[0,1],ymm1[2,3],ymm5[4,5],ymm1[6,7]6283; AVX-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6284; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,2,3,3]6285; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm16286; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,2,2,2]6287; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm06288; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6],ymm0[7]6289; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm4[2,2,3,3]6290; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]6291; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm16292; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]6293; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]6294; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm26295; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3,4],ymm1[5],ymm2[6,7]6296; AVX-NEXT:    vblendps {{.*#+}} ymm14 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]6297; AVX-NEXT:    vmovdqa 112(%r10), %xmm126298; AVX-NEXT:    vmovdqa 112(%rax), %xmm116299; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3]6300; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm8[0,0,0,0]6301; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm8[0,1,0,1]6302; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm06303; AVX-NEXT:    vmovdqa 112(%r9), %xmm106304; AVX-NEXT:    vmovdqa 112(%r8), %xmm76305; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm7[0],xmm10[0],xmm7[1],xmm10[1],xmm7[2],xmm10[2],xmm7[3],xmm10[3]6306; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm13[0,1,0,1]6307; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm13[0],zero,xmm13[1],zero6308; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm16309; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6],ymm0[7]6310; AVX-NEXT:    vmovdqa 112(%rsi), %xmm66311; AVX-NEXT:    vmovdqa 112(%rdi), %xmm56312; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]6313; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]6314; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm16315; AVX-NEXT:    vmovdqa 112(%rcx), %xmm46316; AVX-NEXT:    vmovdqa 112(%rdx), %xmm36317; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]6318; AVX-NEXT:    vpshufd {{.*#+}} xmm15 = xmm0[0,0,1,1]6319; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm15, %ymm156320; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm15[1],ymm1[2,3,4],ymm15[5],ymm1[6,7]6321; AVX-NEXT:    vblendps {{.*#+}} ymm9 = ymm1[0,1],ymm9[2,3],ymm1[4,5],ymm9[6,7]6322; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm13[2,2,3,3]6323; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm13, %ymm16324; AVX-NEXT:    vpshufd {{.*#+}} xmm13 = xmm8[2,2,2,2]6325; AVX-NEXT:    vinsertf128 $1, %xmm8, %ymm13, %ymm86326; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm8[3],ymm1[4,5,6],ymm8[7]6327; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm0[2,2,3,3]6328; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]6329; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm8, %ymm06330; AVX-NEXT:    vpshufd {{.*#+}} xmm8 = xmm2[2,3,2,3]6331; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]6332; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm8, %ymm26333; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0],ymm0[1],ymm2[2,3,4],ymm0[5],ymm2[6,7]6334; AVX-NEXT:    vblendps {{.*#+}} ymm8 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]6335; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]6336; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[0,0,0,0]6337; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[0,1,0,1]6338; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm16339; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm7[4],xmm10[4],xmm7[5],xmm10[5],xmm7[6],xmm10[6],xmm7[7],xmm10[7]6340; AVX-NEXT:    vpshufd {{.*#+}} xmm7 = xmm2[0,1,0,1]6341; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm10 = xmm2[0],zero,xmm2[1],zero6342; AVX-NEXT:    vinsertf128 $1, %xmm10, %ymm7, %ymm76343; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm7[0,1,2],ymm1[3],ymm7[4,5,6],ymm1[7]6344; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]6345; AVX-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]6346; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm5[1,1,1,1]6347; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm46348; AVX-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[0,0,1,1]6349; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm6, %ymm66350; AVX-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0],ymm6[1],ymm4[2,3,4],ymm6[5],ymm4[6,7]6351; AVX-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1],ymm1[2,3],ymm4[4,5],ymm1[6,7]6352; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[2,2,3,3]6353; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm26354; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,2,2,2]6355; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm06356; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3],ymm2[4,5,6],ymm0[7]6357; AVX-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,2,3,3]6358; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]6359; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm26360; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm5[2,3,2,3]6361; AVX-NEXT:    vpshufd {{.*#+}} xmm4 = xmm5[3,3,3,3]6362; AVX-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm36363; AVX-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3,4],ymm2[5],ymm3[6,7]6364; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5],ymm0[6,7]6365; AVX-NEXT:    movq {{[0-9]+}}(%rsp), %rax6366; AVX-NEXT:    vmovaps %ymm0, 992(%rax)6367; AVX-NEXT:    vmovaps %ymm1, 960(%rax)6368; AVX-NEXT:    vmovaps %ymm8, 928(%rax)6369; AVX-NEXT:    vmovaps %ymm9, 896(%rax)6370; AVX-NEXT:    vmovaps %ymm14, 864(%rax)6371; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6372; AVX-NEXT:    vmovaps %ymm0, 832(%rax)6373; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6374; AVX-NEXT:    vmovaps %ymm0, 800(%rax)6375; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6376; AVX-NEXT:    vmovaps %ymm0, 768(%rax)6377; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6378; AVX-NEXT:    vmovaps %ymm0, 736(%rax)6379; AVX-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload6380; AVX-NEXT:    vmovaps %ymm0, 704(%rax)6381; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6382; AVX-NEXT:    vmovaps %ymm0, 672(%rax)6383; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6384; AVX-NEXT:    vmovaps %ymm0, 640(%rax)6385; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6386; AVX-NEXT:    vmovaps %ymm0, 608(%rax)6387; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6388; AVX-NEXT:    vmovaps %ymm0, 576(%rax)6389; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6390; AVX-NEXT:    vmovaps %ymm0, 544(%rax)6391; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6392; AVX-NEXT:    vmovaps %ymm0, 512(%rax)6393; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6394; AVX-NEXT:    vmovaps %ymm0, 480(%rax)6395; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6396; AVX-NEXT:    vmovaps %ymm0, 448(%rax)6397; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6398; AVX-NEXT:    vmovaps %ymm0, 416(%rax)6399; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6400; AVX-NEXT:    vmovaps %ymm0, 384(%rax)6401; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6402; AVX-NEXT:    vmovaps %ymm0, 352(%rax)6403; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6404; AVX-NEXT:    vmovaps %ymm0, 320(%rax)6405; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6406; AVX-NEXT:    vmovaps %ymm0, 288(%rax)6407; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6408; AVX-NEXT:    vmovaps %ymm0, 256(%rax)6409; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6410; AVX-NEXT:    vmovaps %ymm0, 224(%rax)6411; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6412; AVX-NEXT:    vmovaps %ymm0, 192(%rax)6413; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6414; AVX-NEXT:    vmovaps %ymm0, 160(%rax)6415; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6416; AVX-NEXT:    vmovaps %ymm0, 128(%rax)6417; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6418; AVX-NEXT:    vmovaps %ymm0, 96(%rax)6419; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6420; AVX-NEXT:    vmovaps %ymm0, 64(%rax)6421; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6422; AVX-NEXT:    vmovaps %ymm0, 32(%rax)6423; AVX-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6424; AVX-NEXT:    vmovaps %ymm0, (%rax)6425; AVX-NEXT:    addq $744, %rsp # imm = 0x2E86426; AVX-NEXT:    vzeroupper6427; AVX-NEXT:    retq6428;6429; AVX2-LABEL: store_i16_stride8_vf64:6430; AVX2:       # %bb.0:6431; AVX2-NEXT:    subq $712, %rsp # imm = 0x2C86432; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %rax6433; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r106434; AVX2-NEXT:    vmovdqa (%rax), %xmm56435; AVX2-NEXT:    vmovdqa 32(%rax), %xmm06436; AVX2-NEXT:    vmovdqa (%r10), %xmm66437; AVX2-NEXT:    vmovdqa 32(%r10), %xmm16438; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]6439; AVX2-NEXT:    vmovdqa (%r9), %xmm86440; AVX2-NEXT:    vmovdqa (%r8), %xmm96441; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]6442; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]6443; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]6444; AVX2-NEXT:    vmovdqa (%rcx), %xmm106445; AVX2-NEXT:    vmovdqa (%rdx), %xmm116446; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]6447; AVX2-NEXT:    vmovdqa (%rsi), %xmm126448; AVX2-NEXT:    vmovdqa (%rdi), %xmm136449; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]6450; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm15 = xmm14[0],xmm7[0],xmm14[1],xmm7[1]6451; AVX2-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,1,1,3]6452; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm15[0,1],ymm2[2,3],ymm15[4,5],ymm2[6,7]6453; AVX2-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6454; AVX2-NEXT:    vmovdqa 32(%r9), %xmm26455; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm15 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]6456; AVX2-NEXT:    vmovdqa 32(%r8), %xmm46457; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm7 = xmm14[2],xmm7[2],xmm14[3],xmm7[3]6458; AVX2-NEXT:    vmovdqa 32(%rcx), %xmm36459; AVX2-NEXT:    vpermq {{.*#+}} ymm14 = ymm15[0,0,2,1]6460; AVX2-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,1,1,3]6461; AVX2-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1],ymm14[2,3],ymm7[4,5],ymm14[6,7]6462; AVX2-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6463; AVX2-NEXT:    vmovdqa 32(%rdx), %xmm76464; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]6465; AVX2-NEXT:    vmovdqa 32(%rsi), %xmm56466; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]6467; AVX2-NEXT:    vmovdqa 32(%rdi), %xmm86468; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]6469; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm11 = xmm9[0],xmm6[0],xmm9[1],xmm6[1]6470; AVX2-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,0,2,1]6471; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm12 = xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7]6472; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm13 = xmm12[0],xmm10[0],xmm12[1],xmm10[1]6473; AVX2-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,1,1,3]6474; AVX2-NEXT:    vpblendd {{.*#+}} ymm11 = ymm13[0,1],ymm11[2,3],ymm13[4,5],ymm11[6,7]6475; AVX2-NEXT:    vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6476; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm9[2],xmm6[2],xmm9[3],xmm6[3]6477; AVX2-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,2,1]6478; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm9 = xmm12[2],xmm10[2],xmm12[3],xmm10[3]6479; AVX2-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,1,3]6480; AVX2-NEXT:    vpblendd {{.*#+}} ymm6 = ymm9[0,1],ymm6[2,3],ymm9[4,5],ymm6[6,7]6481; AVX2-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6482; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]6483; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]6484; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm10 = xmm9[0],xmm6[0],xmm9[1],xmm6[1]6485; AVX2-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,2,1]6486; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]6487; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]6488; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm13 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]6489; AVX2-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,1,1,3]6490; AVX2-NEXT:    vpblendd {{.*#+}} ymm10 = ymm13[0,1],ymm10[2,3],ymm13[4,5],ymm10[6,7]6491; AVX2-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6492; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm9 = xmm9[2],xmm6[2],xmm9[3],xmm6[3]6493; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm10 = xmm12[2],xmm11[2],xmm12[3],xmm11[3]6494; AVX2-NEXT:    vmovdqa 64(%rax), %xmm66495; AVX2-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,2,1]6496; AVX2-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,1,1,3]6497; AVX2-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0,1],ymm9[2,3],ymm10[4,5],ymm9[6,7]6498; AVX2-NEXT:    vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6499; AVX2-NEXT:    vmovdqa 64(%r10), %xmm96500; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]6501; AVX2-NEXT:    vmovdqa 64(%r9), %xmm06502; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]6503; AVX2-NEXT:    vmovdqa 64(%r8), %xmm16504; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]6505; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm10[0],xmm4[1],xmm10[1]6506; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]6507; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7]6508; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm7 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]6509; AVX2-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,1,1,3]6510; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1],ymm2[2,3],ymm7[4,5],ymm2[6,7]6511; AVX2-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6512; AVX2-NEXT:    vmovdqa 64(%rcx), %xmm26513; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm4[2],xmm10[2],xmm4[3],xmm10[3]6514; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,0,2,1]6515; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm5[2],xmm3[2],xmm5[3],xmm3[3]6516; AVX2-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,1,1,3]6517; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]6518; AVX2-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6519; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3]6520; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]6521; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]6522; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,0,2,1]6523; AVX2-NEXT:    vmovdqa 64(%rdx), %xmm76524; AVX2-NEXT:    vmovdqa 64(%rsi), %xmm86525; AVX2-NEXT:    vmovdqa 64(%rdi), %xmm106526; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]6527; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]6528; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm13 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]6529; AVX2-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,1,1,3]6530; AVX2-NEXT:    vpblendd {{.*#+}} ymm5 = ymm13[0,1],ymm5[2,3],ymm13[4,5],ymm5[6,7]6531; AVX2-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6532; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]6533; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm12[2],xmm11[2],xmm12[3],xmm11[3]6534; AVX2-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]6535; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,1,3]6536; AVX2-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,3],ymm4[4,5],ymm3[6,7]6537; AVX2-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6538; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm9[4],xmm6[4],xmm9[5],xmm6[5],xmm9[6],xmm6[6],xmm9[7],xmm6[7]6539; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]6540; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]6541; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]6542; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]6543; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm10[4],xmm8[4],xmm10[5],xmm8[5],xmm10[6],xmm8[6],xmm10[7],xmm8[7]6544; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]6545; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,1,1,3]6546; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm2[2,3],ymm5[4,5],ymm2[6,7]6547; AVX2-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6548; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]6549; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]6550; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]6551; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]6552; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]6553; AVX2-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6554; AVX2-NEXT:    vmovdqa 96(%rax), %xmm06555; AVX2-NEXT:    vmovdqa 96(%r10), %xmm16556; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]6557; AVX2-NEXT:    vmovdqa 96(%r9), %xmm36558; AVX2-NEXT:    vmovdqa 96(%r8), %xmm46559; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]6560; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]6561; AVX2-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,2,1]6562; AVX2-NEXT:    vmovdqa 96(%rcx), %xmm76563; AVX2-NEXT:    vmovdqa 96(%rdx), %xmm86564; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]6565; AVX2-NEXT:    vmovdqa 96(%rsi), %xmm106566; AVX2-NEXT:    vmovdqa 96(%rdi), %xmm116567; AVX2-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]6568; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm12 = xmm13[0],xmm9[0],xmm13[1],xmm9[1]6569; AVX2-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,1,1,3]6570; AVX2-NEXT:    vpblendd {{.*#+}} ymm6 = ymm12[0,1],ymm6[2,3],ymm12[4,5],ymm6[6,7]6571; AVX2-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6572; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm5[2],xmm2[2],xmm5[3],xmm2[3]6573; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm13[2],xmm9[2],xmm13[3],xmm9[3]6574; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]6575; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,1,1,3]6576; AVX2-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm2[2,3],ymm5[4,5],ymm2[6,7]6577; AVX2-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6578; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]6579; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]6580; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]6581; AVX2-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]6582; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]6583; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,0,2,1]6584; AVX2-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]6585; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,1,1,3]6586; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1],ymm4[2,3],ymm5[4,5],ymm4[6,7]6587; AVX2-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6588; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]6589; AVX2-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]6590; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]6591; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]6592; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]6593; AVX2-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6594; AVX2-NEXT:    vmovdqa (%r8), %ymm116595; AVX2-NEXT:    vmovdqa (%r9), %ymm16596; AVX2-NEXT:    vmovdqa (%r10), %ymm26597; AVX2-NEXT:    vmovdqa (%rax), %ymm36598; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11]6599; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm11[0],ymm1[0],ymm11[1],ymm1[1],ymm11[2],ymm1[2],ymm11[3],ymm1[3],ymm11[8],ymm1[8],ymm11[9],ymm1[9],ymm11[10],ymm1[10],ymm11[11],ymm1[11]6600; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm6 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[4],ymm4[4],ymm5[5],ymm4[5]6601; AVX2-NEXT:    vpermq {{.*#+}} ymm12 = ymm6[0,2,2,3]6602; AVX2-NEXT:    vmovdqa (%rdi), %ymm76603; AVX2-NEXT:    vmovdqa (%rsi), %ymm86604; AVX2-NEXT:    vmovdqa (%rdx), %ymm96605; AVX2-NEXT:    vmovdqa (%rcx), %ymm66606; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm9[0],ymm6[0],ymm9[1],ymm6[1],ymm9[2],ymm6[2],ymm9[3],ymm6[3],ymm9[8],ymm6[8],ymm9[9],ymm6[9],ymm9[10],ymm6[10],ymm9[11],ymm6[11]6607; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]6608; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm13 = ymm0[0],ymm10[0],ymm0[1],ymm10[1],ymm0[4],ymm10[4],ymm0[5],ymm10[5]6609; AVX2-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[2,1,3,3]6610; AVX2-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1],ymm12[2,3],ymm13[4,5],ymm12[6,7]6611; AVX2-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6612; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm4 = ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[6],ymm4[6],ymm5[7],ymm4[7]6613; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm10[2],ymm0[3],ymm10[3],ymm0[6],ymm10[6],ymm0[7],ymm10[7]6614; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3]6615; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3]6616; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm4[2,3],ymm0[4,5],ymm4[6,7]6617; AVX2-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6618; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15]6619; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm11[4],ymm1[4],ymm11[5],ymm1[5],ymm11[6],ymm1[6],ymm11[7],ymm1[7],ymm11[12],ymm1[12],ymm11[13],ymm1[13],ymm11[14],ymm1[14],ymm11[15],ymm1[15]6620; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm9[4],ymm6[4],ymm9[5],ymm6[5],ymm9[6],ymm6[6],ymm9[7],ymm6[7],ymm9[12],ymm6[12],ymm9[13],ymm6[13],ymm9[14],ymm6[14],ymm9[15],ymm6[15]6621; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15]6622; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm4 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5]6623; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3]6624; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm5 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[4],ymm2[4],ymm3[5],ymm2[5]6625; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,1,3,3]6626; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1],ymm4[2,3],ymm5[4,5],ymm4[6,7]6627; AVX2-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6628; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[6],ymm0[6],ymm1[7],ymm0[7]6629; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm1 = ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[6],ymm2[6],ymm3[7],ymm2[7]6630; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]6631; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,3,3]6632; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]6633; AVX2-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6634; AVX2-NEXT:    vmovdqa 32(%r8), %ymm96635; AVX2-NEXT:    vmovdqa 32(%r9), %ymm86636; AVX2-NEXT:    vmovdqa 32(%r10), %ymm76637; AVX2-NEXT:    vmovdqa 32(%rax), %ymm36638; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm7[0],ymm3[0],ymm7[1],ymm3[1],ymm7[2],ymm3[2],ymm7[3],ymm3[3],ymm7[8],ymm3[8],ymm7[9],ymm3[9],ymm7[10],ymm3[10],ymm7[11],ymm3[11]6639; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]6640; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm6 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[4],ymm4[4],ymm5[5],ymm4[5]6641; AVX2-NEXT:    vpermq {{.*#+}} ymm10 = ymm6[0,2,2,3]6642; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm116643; AVX2-NEXT:    vmovdqa 32(%rsi), %ymm136644; AVX2-NEXT:    vmovdqa 32(%rdx), %ymm26645; AVX2-NEXT:    vmovdqa 32(%rcx), %ymm16646; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]6647; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm11[0],ymm13[0],ymm11[1],ymm13[1],ymm11[2],ymm13[2],ymm11[3],ymm13[3],ymm11[8],ymm13[8],ymm11[9],ymm13[9],ymm11[10],ymm13[10],ymm11[11],ymm13[11]6648; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm15 = ymm0[0],ymm6[0],ymm0[1],ymm6[1],ymm0[4],ymm6[4],ymm0[5],ymm6[5]6649; AVX2-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[2,1,3,3]6650; AVX2-NEXT:    vpblendd {{.*#+}} ymm10 = ymm15[0,1],ymm10[2,3],ymm15[4,5],ymm10[6,7]6651; AVX2-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6652; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm4 = ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[6],ymm4[6],ymm5[7],ymm4[7]6653; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm6[2],ymm0[3],ymm6[3],ymm0[6],ymm6[6],ymm0[7],ymm6[7]6654; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3]6655; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3]6656; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm4[2,3],ymm0[4,5],ymm4[6,7]6657; AVX2-NEXT:    vmovdqu %ymm0, (%rsp) # 32-byte Spill6658; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm7[4],ymm3[4],ymm7[5],ymm3[5],ymm7[6],ymm3[6],ymm7[7],ymm3[7],ymm7[12],ymm3[12],ymm7[13],ymm3[13],ymm7[14],ymm3[14],ymm7[15],ymm3[15]6659; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15]6660; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15]6661; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm11[4],ymm13[4],ymm11[5],ymm13[5],ymm11[6],ymm13[6],ymm11[7],ymm13[7],ymm11[12],ymm13[12],ymm11[13],ymm13[13],ymm11[14],ymm13[14],ymm11[15],ymm13[15]6662; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm4 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[4],ymm0[4],ymm3[5],ymm0[5]6663; AVX2-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3]6664; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm5 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[4],ymm1[4],ymm2[5],ymm1[5]6665; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,1,3,3]6666; AVX2-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1],ymm4[2,3],ymm5[4,5],ymm4[6,7]6667; AVX2-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6668; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[6],ymm0[6],ymm3[7],ymm0[7]6669; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm1 = ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[6],ymm1[6],ymm2[7],ymm1[7]6670; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]6671; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,3,3]6672; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]6673; AVX2-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6674; AVX2-NEXT:    vmovdqa 64(%r8), %ymm56675; AVX2-NEXT:    vmovdqa 64(%r9), %ymm46676; AVX2-NEXT:    vmovdqa 64(%r10), %ymm76677; AVX2-NEXT:    vmovdqa 64(%rax), %ymm86678; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]6679; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[8],ymm4[8],ymm5[9],ymm4[9],ymm5[10],ymm4[10],ymm5[11],ymm4[11]6680; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm3 = ymm9[0],ymm2[0],ymm9[1],ymm2[1],ymm9[4],ymm2[4],ymm9[5],ymm2[5]6681; AVX2-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[0,2,2,3]6682; AVX2-NEXT:    vmovdqa 64(%rdi), %ymm116683; AVX2-NEXT:    vmovdqa 64(%rsi), %ymm136684; AVX2-NEXT:    vmovdqa 64(%rdx), %ymm156685; AVX2-NEXT:    vmovdqa 64(%rcx), %ymm36686; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm15[0],ymm3[0],ymm15[1],ymm3[1],ymm15[2],ymm3[2],ymm15[3],ymm3[3],ymm15[8],ymm3[8],ymm15[9],ymm3[9],ymm15[10],ymm3[10],ymm15[11],ymm3[11]6687; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm11[0],ymm13[0],ymm11[1],ymm13[1],ymm11[2],ymm13[2],ymm11[3],ymm13[3],ymm11[8],ymm13[8],ymm11[9],ymm13[9],ymm11[10],ymm13[10],ymm11[11],ymm13[11]6688; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm14 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]6689; AVX2-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[2,1,3,3]6690; AVX2-NEXT:    vpblendd {{.*#+}} ymm6 = ymm14[0,1],ymm6[2,3],ymm14[4,5],ymm6[6,7]6691; AVX2-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6692; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm9[2],ymm2[2],ymm9[3],ymm2[3],ymm9[6],ymm2[6],ymm9[7],ymm2[7]6693; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]6694; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm2[0,2,2,3]6695; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3]6696; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]6697; AVX2-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6698; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15]6699; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm5[4],ymm4[4],ymm5[5],ymm4[5],ymm5[6],ymm4[6],ymm5[7],ymm4[7],ymm5[12],ymm4[12],ymm5[13],ymm4[13],ymm5[14],ymm4[14],ymm5[15],ymm4[15]6700; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm15[4],ymm3[4],ymm15[5],ymm3[5],ymm15[6],ymm3[6],ymm15[7],ymm3[7],ymm15[12],ymm3[12],ymm15[13],ymm3[13],ymm15[14],ymm3[14],ymm15[15],ymm3[15]6701; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm11[4],ymm13[4],ymm11[5],ymm13[5],ymm11[6],ymm13[6],ymm11[7],ymm13[7],ymm11[12],ymm13[12],ymm11[13],ymm13[13],ymm11[14],ymm13[14],ymm11[15],ymm13[15]6702; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm1 = ymm4[0],ymm0[0],ymm4[1],ymm0[1],ymm4[4],ymm0[4],ymm4[5],ymm0[5]6703; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]6704; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm7 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[4],ymm3[4],ymm5[5],ymm3[5]6705; AVX2-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,1,3,3]6706; AVX2-NEXT:    vpblendd {{.*#+}} ymm10 = ymm7[0,1],ymm1[2,3],ymm7[4,5],ymm1[6,7]6707; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm4[2],ymm0[2],ymm4[3],ymm0[3],ymm4[6],ymm0[6],ymm4[7],ymm0[7]6708; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm3 = ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[6],ymm3[6],ymm5[7],ymm3[7]6709; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]6710; AVX2-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,1,3,3]6711; AVX2-NEXT:    vpblendd {{.*#+}} ymm6 = ymm3[0,1],ymm0[2,3],ymm3[4,5],ymm0[6,7]6712; AVX2-NEXT:    vmovdqa 96(%r8), %ymm36713; AVX2-NEXT:    vmovdqa 96(%r9), %ymm46714; AVX2-NEXT:    vmovdqa 96(%r10), %ymm56715; AVX2-NEXT:    vmovdqa 96(%rax), %ymm76716; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm5[0],ymm7[0],ymm5[1],ymm7[1],ymm5[2],ymm7[2],ymm5[3],ymm7[3],ymm5[8],ymm7[8],ymm5[9],ymm7[9],ymm5[10],ymm7[10],ymm5[11],ymm7[11]6717; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11]6718; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm11 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[4],ymm8[4],ymm9[5],ymm8[5]6719; AVX2-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,2,2,3]6720; AVX2-NEXT:    vmovdqa 96(%rdi), %ymm136721; AVX2-NEXT:    vmovdqa 96(%rsi), %ymm146722; AVX2-NEXT:    vmovdqa 96(%rdx), %ymm156723; AVX2-NEXT:    vmovdqa 96(%rcx), %ymm26724; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm15[0],ymm2[0],ymm15[1],ymm2[1],ymm15[2],ymm2[2],ymm15[3],ymm2[3],ymm15[8],ymm2[8],ymm15[9],ymm2[9],ymm15[10],ymm2[10],ymm15[11],ymm2[11]6725; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm13[0],ymm14[0],ymm13[1],ymm14[1],ymm13[2],ymm14[2],ymm13[3],ymm14[3],ymm13[8],ymm14[8],ymm13[9],ymm14[9],ymm13[10],ymm14[10],ymm13[11],ymm14[11]6726; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm12 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]6727; AVX2-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,1,3,3]6728; AVX2-NEXT:    vpblendd {{.*#+}} ymm11 = ymm12[0,1],ymm11[2,3],ymm12[4,5],ymm11[6,7]6729; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm8 = ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[6],ymm8[6],ymm9[7],ymm8[7]6730; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]6731; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm8[0,2,2,3]6732; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3]6733; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]6734; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm5[4],ymm7[4],ymm5[5],ymm7[5],ymm5[6],ymm7[6],ymm5[7],ymm7[7],ymm5[12],ymm7[12],ymm5[13],ymm7[13],ymm5[14],ymm7[14],ymm5[15],ymm7[15]6735; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm4[4],ymm3[5],ymm4[5],ymm3[6],ymm4[6],ymm3[7],ymm4[7],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15]6736; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm15[4],ymm2[4],ymm15[5],ymm2[5],ymm15[6],ymm2[6],ymm15[7],ymm2[7],ymm15[12],ymm2[12],ymm15[13],ymm2[13],ymm15[14],ymm2[14],ymm15[15],ymm2[15]6737; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm13[4],ymm14[4],ymm13[5],ymm14[5],ymm13[6],ymm14[6],ymm13[7],ymm14[7],ymm13[12],ymm14[12],ymm13[13],ymm14[13],ymm13[14],ymm14[14],ymm13[15],ymm14[15]6738; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm5 = ymm3[0],ymm1[0],ymm3[1],ymm1[1],ymm3[4],ymm1[4],ymm3[5],ymm1[5]6739; AVX2-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,2,2,3]6740; AVX2-NEXT:    vpunpckldq {{.*#+}} ymm7 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[4],ymm2[4],ymm4[5],ymm2[5]6741; AVX2-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,1,3,3]6742; AVX2-NEXT:    vpblendd {{.*#+}} ymm5 = ymm7[0,1],ymm5[2,3],ymm7[4,5],ymm5[6,7]6743; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm1 = ymm3[2],ymm1[2],ymm3[3],ymm1[3],ymm3[6],ymm1[6],ymm3[7],ymm1[7]6744; AVX2-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm4[2],ymm2[2],ymm4[3],ymm2[3],ymm4[6],ymm2[6],ymm4[7],ymm2[7]6745; AVX2-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]6746; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,1,3,3]6747; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5],ymm1[6,7]6748; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %rax6749; AVX2-NEXT:    vmovdqa %ymm1, 992(%rax)6750; AVX2-NEXT:    vmovdqa %ymm5, 960(%rax)6751; AVX2-NEXT:    vmovdqa %ymm0, 928(%rax)6752; AVX2-NEXT:    vmovdqa %ymm11, 896(%rax)6753; AVX2-NEXT:    vmovdqa %ymm6, 736(%rax)6754; AVX2-NEXT:    vmovdqa %ymm10, 704(%rax)6755; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6756; AVX2-NEXT:    vmovaps %ymm0, 672(%rax)6757; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6758; AVX2-NEXT:    vmovaps %ymm0, 640(%rax)6759; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6760; AVX2-NEXT:    vmovaps %ymm0, 480(%rax)6761; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6762; AVX2-NEXT:    vmovaps %ymm0, 448(%rax)6763; AVX2-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload6764; AVX2-NEXT:    vmovaps %ymm0, 416(%rax)6765; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6766; AVX2-NEXT:    vmovaps %ymm0, 384(%rax)6767; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6768; AVX2-NEXT:    vmovaps %ymm0, 224(%rax)6769; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6770; AVX2-NEXT:    vmovaps %ymm0, 192(%rax)6771; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6772; AVX2-NEXT:    vmovaps %ymm0, 160(%rax)6773; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6774; AVX2-NEXT:    vmovaps %ymm0, 128(%rax)6775; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6776; AVX2-NEXT:    vmovaps %ymm0, 864(%rax)6777; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6778; AVX2-NEXT:    vmovaps %ymm0, 832(%rax)6779; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6780; AVX2-NEXT:    vmovaps %ymm0, 800(%rax)6781; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6782; AVX2-NEXT:    vmovaps %ymm0, 768(%rax)6783; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6784; AVX2-NEXT:    vmovaps %ymm0, 608(%rax)6785; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6786; AVX2-NEXT:    vmovaps %ymm0, 576(%rax)6787; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6788; AVX2-NEXT:    vmovaps %ymm0, 544(%rax)6789; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6790; AVX2-NEXT:    vmovaps %ymm0, 512(%rax)6791; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6792; AVX2-NEXT:    vmovaps %ymm0, 352(%rax)6793; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6794; AVX2-NEXT:    vmovaps %ymm0, 320(%rax)6795; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6796; AVX2-NEXT:    vmovaps %ymm0, 288(%rax)6797; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6798; AVX2-NEXT:    vmovaps %ymm0, 256(%rax)6799; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6800; AVX2-NEXT:    vmovaps %ymm0, 96(%rax)6801; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6802; AVX2-NEXT:    vmovaps %ymm0, 64(%rax)6803; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6804; AVX2-NEXT:    vmovaps %ymm0, 32(%rax)6805; AVX2-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload6806; AVX2-NEXT:    vmovaps %ymm0, (%rax)6807; AVX2-NEXT:    addq $712, %rsp # imm = 0x2C86808; AVX2-NEXT:    vzeroupper6809; AVX2-NEXT:    retq6810;6811; AVX2-FP-LABEL: store_i16_stride8_vf64:6812; AVX2-FP:       # %bb.0:6813; AVX2-FP-NEXT:    subq $712, %rsp # imm = 0x2C86814; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %rax6815; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %r106816; AVX2-FP-NEXT:    vmovdqa (%rax), %xmm56817; AVX2-FP-NEXT:    vmovdqa 32(%rax), %xmm06818; AVX2-FP-NEXT:    vmovdqa (%r10), %xmm66819; AVX2-FP-NEXT:    vmovdqa 32(%r10), %xmm16820; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]6821; AVX2-FP-NEXT:    vmovdqa (%r9), %xmm86822; AVX2-FP-NEXT:    vmovdqa (%r8), %xmm96823; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]6824; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]6825; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]6826; AVX2-FP-NEXT:    vmovdqa (%rcx), %xmm106827; AVX2-FP-NEXT:    vmovdqa (%rdx), %xmm116828; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]6829; AVX2-FP-NEXT:    vmovdqa (%rsi), %xmm126830; AVX2-FP-NEXT:    vmovdqa (%rdi), %xmm136831; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]6832; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm15 = xmm14[0],xmm7[0],xmm14[1],xmm7[1]6833; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[0,1,1,3]6834; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm15[0,1],ymm2[2,3],ymm15[4,5],ymm2[6,7]6835; AVX2-FP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6836; AVX2-FP-NEXT:    vmovdqa 32(%r9), %xmm26837; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm15 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]6838; AVX2-FP-NEXT:    vmovdqa 32(%r8), %xmm46839; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm7 = xmm14[2],xmm7[2],xmm14[3],xmm7[3]6840; AVX2-FP-NEXT:    vmovdqa 32(%rcx), %xmm36841; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm14 = ymm15[0,0,2,1]6842; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,1,1,3]6843; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1],ymm14[2,3],ymm7[4,5],ymm14[6,7]6844; AVX2-FP-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6845; AVX2-FP-NEXT:    vmovdqa 32(%rdx), %xmm76846; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]6847; AVX2-FP-NEXT:    vmovdqa 32(%rsi), %xmm56848; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm9 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]6849; AVX2-FP-NEXT:    vmovdqa 32(%rdi), %xmm86850; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]6851; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm11 = xmm9[0],xmm6[0],xmm9[1],xmm6[1]6852; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,0,2,1]6853; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm12 = xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7]6854; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm13 = xmm12[0],xmm10[0],xmm12[1],xmm10[1]6855; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,1,1,3]6856; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm11 = ymm13[0,1],ymm11[2,3],ymm13[4,5],ymm11[6,7]6857; AVX2-FP-NEXT:    vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6858; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm9[2],xmm6[2],xmm9[3],xmm6[3]6859; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,2,1]6860; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm9 = xmm12[2],xmm10[2],xmm12[3],xmm10[3]6861; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,1,3]6862; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm9[0,1],ymm6[2,3],ymm9[4,5],ymm6[6,7]6863; AVX2-FP-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6864; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]6865; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]6866; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm10 = xmm9[0],xmm6[0],xmm9[1],xmm6[1]6867; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,2,1]6868; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3]6869; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]6870; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm13 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]6871; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,1,1,3]6872; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm13[0,1],ymm10[2,3],ymm13[4,5],ymm10[6,7]6873; AVX2-FP-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6874; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm9 = xmm9[2],xmm6[2],xmm9[3],xmm6[3]6875; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm10 = xmm12[2],xmm11[2],xmm12[3],xmm11[3]6876; AVX2-FP-NEXT:    vmovdqa 64(%rax), %xmm66877; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,0,2,1]6878; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,1,1,3]6879; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0,1],ymm9[2,3],ymm10[4,5],ymm9[6,7]6880; AVX2-FP-NEXT:    vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6881; AVX2-FP-NEXT:    vmovdqa 64(%r10), %xmm96882; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]6883; AVX2-FP-NEXT:    vmovdqa 64(%r9), %xmm06884; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]6885; AVX2-FP-NEXT:    vmovdqa 64(%r8), %xmm16886; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]6887; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm10[0],xmm4[1],xmm10[1]6888; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]6889; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7]6890; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm7 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]6891; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,1,1,3]6892; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1],ymm2[2,3],ymm7[4,5],ymm2[6,7]6893; AVX2-FP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6894; AVX2-FP-NEXT:    vmovdqa 64(%rcx), %xmm26895; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm4[2],xmm10[2],xmm4[3],xmm10[3]6896; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,0,2,1]6897; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm5[2],xmm3[2],xmm5[3],xmm3[3]6898; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,1,1,3]6899; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3],ymm3[4,5],ymm4[6,7]6900; AVX2-FP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6901; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3]6902; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]6903; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]6904; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,0,2,1]6905; AVX2-FP-NEXT:    vmovdqa 64(%rdx), %xmm76906; AVX2-FP-NEXT:    vmovdqa 64(%rsi), %xmm86907; AVX2-FP-NEXT:    vmovdqa 64(%rdi), %xmm106908; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]6909; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3]6910; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm13 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]6911; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,1,1,3]6912; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm13[0,1],ymm5[2,3],ymm13[4,5],ymm5[6,7]6913; AVX2-FP-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6914; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]6915; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm12[2],xmm11[2],xmm12[3],xmm11[3]6916; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,2,1]6917; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,1,3]6918; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,3],ymm4[4,5],ymm3[6,7]6919; AVX2-FP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6920; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm9[4],xmm6[4],xmm9[5],xmm6[5],xmm9[6],xmm6[6],xmm9[7],xmm6[7]6921; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]6922; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]6923; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]6924; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]6925; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm10[4],xmm8[4],xmm10[5],xmm8[5],xmm10[6],xmm8[6],xmm10[7],xmm8[7]6926; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]6927; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,1,1,3]6928; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm2[2,3],ymm5[4,5],ymm2[6,7]6929; AVX2-FP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6930; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]6931; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]6932; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]6933; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]6934; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]6935; AVX2-FP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6936; AVX2-FP-NEXT:    vmovdqa 96(%rax), %xmm06937; AVX2-FP-NEXT:    vmovdqa 96(%r10), %xmm16938; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]6939; AVX2-FP-NEXT:    vmovdqa 96(%r9), %xmm36940; AVX2-FP-NEXT:    vmovdqa 96(%r8), %xmm46941; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]6942; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]6943; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,0,2,1]6944; AVX2-FP-NEXT:    vmovdqa 96(%rcx), %xmm76945; AVX2-FP-NEXT:    vmovdqa 96(%rdx), %xmm86946; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]6947; AVX2-FP-NEXT:    vmovdqa 96(%rsi), %xmm106948; AVX2-FP-NEXT:    vmovdqa 96(%rdi), %xmm116949; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]6950; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm12 = xmm13[0],xmm9[0],xmm13[1],xmm9[1]6951; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,1,1,3]6952; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm12[0,1],ymm6[2,3],ymm12[4,5],ymm6[6,7]6953; AVX2-FP-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6954; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm5[2],xmm2[2],xmm5[3],xmm2[3]6955; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm13[2],xmm9[2],xmm13[3],xmm9[3]6956; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,0,2,1]6957; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,1,1,3]6958; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm2[2,3],ymm5[4,5],ymm2[6,7]6959; AVX2-FP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6960; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]6961; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]6962; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]6963; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]6964; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]6965; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,0,2,1]6966; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]6967; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,1,1,3]6968; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1],ymm4[2,3],ymm5[4,5],ymm4[6,7]6969; AVX2-FP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6970; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm1[2],xmm0[2],xmm1[3],xmm0[3]6971; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]6972; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,2,1]6973; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]6974; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]6975; AVX2-FP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6976; AVX2-FP-NEXT:    vmovdqa (%r8), %ymm116977; AVX2-FP-NEXT:    vmovdqa (%r9), %ymm16978; AVX2-FP-NEXT:    vmovdqa (%r10), %ymm26979; AVX2-FP-NEXT:    vmovdqa (%rax), %ymm36980; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11]6981; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm11[0],ymm1[0],ymm11[1],ymm1[1],ymm11[2],ymm1[2],ymm11[3],ymm1[3],ymm11[8],ymm1[8],ymm11[9],ymm1[9],ymm11[10],ymm1[10],ymm11[11],ymm1[11]6982; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm6 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[4],ymm4[4],ymm5[5],ymm4[5]6983; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm12 = ymm6[0,2,2,3]6984; AVX2-FP-NEXT:    vmovdqa (%rdi), %ymm76985; AVX2-FP-NEXT:    vmovdqa (%rsi), %ymm86986; AVX2-FP-NEXT:    vmovdqa (%rdx), %ymm96987; AVX2-FP-NEXT:    vmovdqa (%rcx), %ymm66988; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm9[0],ymm6[0],ymm9[1],ymm6[1],ymm9[2],ymm6[2],ymm9[3],ymm6[3],ymm9[8],ymm6[8],ymm9[9],ymm6[9],ymm9[10],ymm6[10],ymm9[11],ymm6[11]6989; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]6990; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm13 = ymm0[0],ymm10[0],ymm0[1],ymm10[1],ymm0[4],ymm10[4],ymm0[5],ymm10[5]6991; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[2,1,3,3]6992; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1],ymm12[2,3],ymm13[4,5],ymm12[6,7]6993; AVX2-FP-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill6994; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm4 = ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[6],ymm4[6],ymm5[7],ymm4[7]6995; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm10[2],ymm0[3],ymm10[3],ymm0[6],ymm10[6],ymm0[7],ymm10[7]6996; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3]6997; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3]6998; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm4[2,3],ymm0[4,5],ymm4[6,7]6999; AVX2-FP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7000; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15]7001; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm11[4],ymm1[4],ymm11[5],ymm1[5],ymm11[6],ymm1[6],ymm11[7],ymm1[7],ymm11[12],ymm1[12],ymm11[13],ymm1[13],ymm11[14],ymm1[14],ymm11[15],ymm1[15]7002; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm9[4],ymm6[4],ymm9[5],ymm6[5],ymm9[6],ymm6[6],ymm9[7],ymm6[7],ymm9[12],ymm6[12],ymm9[13],ymm6[13],ymm9[14],ymm6[14],ymm9[15],ymm6[15]7003; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15]7004; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm4 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[4],ymm0[4],ymm1[5],ymm0[5]7005; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3]7006; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm5 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[4],ymm2[4],ymm3[5],ymm2[5]7007; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,1,3,3]7008; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1],ymm4[2,3],ymm5[4,5],ymm4[6,7]7009; AVX2-FP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7010; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[6],ymm0[6],ymm1[7],ymm0[7]7011; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm1 = ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[6],ymm2[6],ymm3[7],ymm2[7]7012; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]7013; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,3,3]7014; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]7015; AVX2-FP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7016; AVX2-FP-NEXT:    vmovdqa 32(%r8), %ymm97017; AVX2-FP-NEXT:    vmovdqa 32(%r9), %ymm87018; AVX2-FP-NEXT:    vmovdqa 32(%r10), %ymm77019; AVX2-FP-NEXT:    vmovdqa 32(%rax), %ymm37020; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm7[0],ymm3[0],ymm7[1],ymm3[1],ymm7[2],ymm3[2],ymm7[3],ymm3[3],ymm7[8],ymm3[8],ymm7[9],ymm3[9],ymm7[10],ymm3[10],ymm7[11],ymm3[11]7021; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]7022; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm6 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[4],ymm4[4],ymm5[5],ymm4[5]7023; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm10 = ymm6[0,2,2,3]7024; AVX2-FP-NEXT:    vmovdqa 32(%rdi), %ymm117025; AVX2-FP-NEXT:    vmovdqa 32(%rsi), %ymm137026; AVX2-FP-NEXT:    vmovdqa 32(%rdx), %ymm27027; AVX2-FP-NEXT:    vmovdqa 32(%rcx), %ymm17028; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11]7029; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm11[0],ymm13[0],ymm11[1],ymm13[1],ymm11[2],ymm13[2],ymm11[3],ymm13[3],ymm11[8],ymm13[8],ymm11[9],ymm13[9],ymm11[10],ymm13[10],ymm11[11],ymm13[11]7030; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm15 = ymm0[0],ymm6[0],ymm0[1],ymm6[1],ymm0[4],ymm6[4],ymm0[5],ymm6[5]7031; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[2,1,3,3]7032; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm15[0,1],ymm10[2,3],ymm15[4,5],ymm10[6,7]7033; AVX2-FP-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7034; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm4 = ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[6],ymm4[6],ymm5[7],ymm4[7]7035; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm6[2],ymm0[3],ymm6[3],ymm0[6],ymm6[6],ymm0[7],ymm6[7]7036; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3]7037; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3]7038; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm4[2,3],ymm0[4,5],ymm4[6,7]7039; AVX2-FP-NEXT:    vmovdqu %ymm0, (%rsp) # 32-byte Spill7040; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm7[4],ymm3[4],ymm7[5],ymm3[5],ymm7[6],ymm3[6],ymm7[7],ymm3[7],ymm7[12],ymm3[12],ymm7[13],ymm3[13],ymm7[14],ymm3[14],ymm7[15],ymm3[15]7041; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15]7042; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15]7043; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm11[4],ymm13[4],ymm11[5],ymm13[5],ymm11[6],ymm13[6],ymm11[7],ymm13[7],ymm11[12],ymm13[12],ymm11[13],ymm13[13],ymm11[14],ymm13[14],ymm11[15],ymm13[15]7044; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm4 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[4],ymm0[4],ymm3[5],ymm0[5]7045; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3]7046; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm5 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[4],ymm1[4],ymm2[5],ymm1[5]7047; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,1,3,3]7048; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1],ymm4[2,3],ymm5[4,5],ymm4[6,7]7049; AVX2-FP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7050; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[6],ymm0[6],ymm3[7],ymm0[7]7051; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm1 = ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[6],ymm1[6],ymm2[7],ymm1[7]7052; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]7053; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,1,3,3]7054; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]7055; AVX2-FP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7056; AVX2-FP-NEXT:    vmovdqa 64(%r8), %ymm57057; AVX2-FP-NEXT:    vmovdqa 64(%r9), %ymm47058; AVX2-FP-NEXT:    vmovdqa 64(%r10), %ymm77059; AVX2-FP-NEXT:    vmovdqa 64(%rax), %ymm87060; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]7061; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm5[0],ymm4[0],ymm5[1],ymm4[1],ymm5[2],ymm4[2],ymm5[3],ymm4[3],ymm5[8],ymm4[8],ymm5[9],ymm4[9],ymm5[10],ymm4[10],ymm5[11],ymm4[11]7062; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm3 = ymm9[0],ymm2[0],ymm9[1],ymm2[1],ymm9[4],ymm2[4],ymm9[5],ymm2[5]7063; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[0,2,2,3]7064; AVX2-FP-NEXT:    vmovdqa 64(%rdi), %ymm117065; AVX2-FP-NEXT:    vmovdqa 64(%rsi), %ymm137066; AVX2-FP-NEXT:    vmovdqa 64(%rdx), %ymm157067; AVX2-FP-NEXT:    vmovdqa 64(%rcx), %ymm37068; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm15[0],ymm3[0],ymm15[1],ymm3[1],ymm15[2],ymm3[2],ymm15[3],ymm3[3],ymm15[8],ymm3[8],ymm15[9],ymm3[9],ymm15[10],ymm3[10],ymm15[11],ymm3[11]7069; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm11[0],ymm13[0],ymm11[1],ymm13[1],ymm11[2],ymm13[2],ymm11[3],ymm13[3],ymm11[8],ymm13[8],ymm11[9],ymm13[9],ymm11[10],ymm13[10],ymm11[11],ymm13[11]7070; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm14 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]7071; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[2,1,3,3]7072; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm14[0,1],ymm6[2,3],ymm14[4,5],ymm6[6,7]7073; AVX2-FP-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7074; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm9[2],ymm2[2],ymm9[3],ymm2[3],ymm9[6],ymm2[6],ymm9[7],ymm2[7]7075; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]7076; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm2[0,2,2,3]7077; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3]7078; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]7079; AVX2-FP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7080; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15]7081; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm5[4],ymm4[4],ymm5[5],ymm4[5],ymm5[6],ymm4[6],ymm5[7],ymm4[7],ymm5[12],ymm4[12],ymm5[13],ymm4[13],ymm5[14],ymm4[14],ymm5[15],ymm4[15]7082; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm15[4],ymm3[4],ymm15[5],ymm3[5],ymm15[6],ymm3[6],ymm15[7],ymm3[7],ymm15[12],ymm3[12],ymm15[13],ymm3[13],ymm15[14],ymm3[14],ymm15[15],ymm3[15]7083; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm11[4],ymm13[4],ymm11[5],ymm13[5],ymm11[6],ymm13[6],ymm11[7],ymm13[7],ymm11[12],ymm13[12],ymm11[13],ymm13[13],ymm11[14],ymm13[14],ymm11[15],ymm13[15]7084; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm1 = ymm4[0],ymm0[0],ymm4[1],ymm0[1],ymm4[4],ymm0[4],ymm4[5],ymm0[5]7085; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]7086; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm7 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[4],ymm3[4],ymm5[5],ymm3[5]7087; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,1,3,3]7088; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm7[0,1],ymm1[2,3],ymm7[4,5],ymm1[6,7]7089; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm4[2],ymm0[2],ymm4[3],ymm0[3],ymm4[6],ymm0[6],ymm4[7],ymm0[7]7090; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm3 = ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[6],ymm3[6],ymm5[7],ymm3[7]7091; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]7092; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,1,3,3]7093; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm3[0,1],ymm0[2,3],ymm3[4,5],ymm0[6,7]7094; AVX2-FP-NEXT:    vmovdqa 96(%r8), %ymm37095; AVX2-FP-NEXT:    vmovdqa 96(%r9), %ymm47096; AVX2-FP-NEXT:    vmovdqa 96(%r10), %ymm57097; AVX2-FP-NEXT:    vmovdqa 96(%rax), %ymm77098; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm5[0],ymm7[0],ymm5[1],ymm7[1],ymm5[2],ymm7[2],ymm5[3],ymm7[3],ymm5[8],ymm7[8],ymm5[9],ymm7[9],ymm5[10],ymm7[10],ymm5[11],ymm7[11]7099; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11]7100; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm11 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[4],ymm8[4],ymm9[5],ymm8[5]7101; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[0,2,2,3]7102; AVX2-FP-NEXT:    vmovdqa 96(%rdi), %ymm137103; AVX2-FP-NEXT:    vmovdqa 96(%rsi), %ymm147104; AVX2-FP-NEXT:    vmovdqa 96(%rdx), %ymm157105; AVX2-FP-NEXT:    vmovdqa 96(%rcx), %ymm27106; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm15[0],ymm2[0],ymm15[1],ymm2[1],ymm15[2],ymm2[2],ymm15[3],ymm2[3],ymm15[8],ymm2[8],ymm15[9],ymm2[9],ymm15[10],ymm2[10],ymm15[11],ymm2[11]7107; AVX2-FP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm13[0],ymm14[0],ymm13[1],ymm14[1],ymm13[2],ymm14[2],ymm13[3],ymm14[3],ymm13[8],ymm14[8],ymm13[9],ymm14[9],ymm13[10],ymm14[10],ymm13[11],ymm14[11]7108; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm12 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]7109; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,1,3,3]7110; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm11 = ymm12[0,1],ymm11[2,3],ymm12[4,5],ymm11[6,7]7111; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm8 = ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[6],ymm8[6],ymm9[7],ymm8[7]7112; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]7113; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm8[0,2,2,3]7114; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,3,3]7115; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]7116; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm5[4],ymm7[4],ymm5[5],ymm7[5],ymm5[6],ymm7[6],ymm5[7],ymm7[7],ymm5[12],ymm7[12],ymm5[13],ymm7[13],ymm5[14],ymm7[14],ymm5[15],ymm7[15]7117; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm4[4],ymm3[5],ymm4[5],ymm3[6],ymm4[6],ymm3[7],ymm4[7],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15]7118; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm15[4],ymm2[4],ymm15[5],ymm2[5],ymm15[6],ymm2[6],ymm15[7],ymm2[7],ymm15[12],ymm2[12],ymm15[13],ymm2[13],ymm15[14],ymm2[14],ymm15[15],ymm2[15]7119; AVX2-FP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm13[4],ymm14[4],ymm13[5],ymm14[5],ymm13[6],ymm14[6],ymm13[7],ymm14[7],ymm13[12],ymm14[12],ymm13[13],ymm14[13],ymm13[14],ymm14[14],ymm13[15],ymm14[15]7120; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm5 = ymm3[0],ymm1[0],ymm3[1],ymm1[1],ymm3[4],ymm1[4],ymm3[5],ymm1[5]7121; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[0,2,2,3]7122; AVX2-FP-NEXT:    vpunpckldq {{.*#+}} ymm7 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[4],ymm2[4],ymm4[5],ymm2[5]7123; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,1,3,3]7124; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm7[0,1],ymm5[2,3],ymm7[4,5],ymm5[6,7]7125; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm1 = ymm3[2],ymm1[2],ymm3[3],ymm1[3],ymm3[6],ymm1[6],ymm3[7],ymm1[7]7126; AVX2-FP-NEXT:    vpunpckhdq {{.*#+}} ymm2 = ymm4[2],ymm2[2],ymm4[3],ymm2[3],ymm4[6],ymm2[6],ymm4[7],ymm2[7]7127; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3]7128; AVX2-FP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,1,3,3]7129; AVX2-FP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5],ymm1[6,7]7130; AVX2-FP-NEXT:    movq {{[0-9]+}}(%rsp), %rax7131; AVX2-FP-NEXT:    vmovdqa %ymm1, 992(%rax)7132; AVX2-FP-NEXT:    vmovdqa %ymm5, 960(%rax)7133; AVX2-FP-NEXT:    vmovdqa %ymm0, 928(%rax)7134; AVX2-FP-NEXT:    vmovdqa %ymm11, 896(%rax)7135; AVX2-FP-NEXT:    vmovdqa %ymm6, 736(%rax)7136; AVX2-FP-NEXT:    vmovdqa %ymm10, 704(%rax)7137; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7138; AVX2-FP-NEXT:    vmovaps %ymm0, 672(%rax)7139; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7140; AVX2-FP-NEXT:    vmovaps %ymm0, 640(%rax)7141; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7142; AVX2-FP-NEXT:    vmovaps %ymm0, 480(%rax)7143; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7144; AVX2-FP-NEXT:    vmovaps %ymm0, 448(%rax)7145; AVX2-FP-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload7146; AVX2-FP-NEXT:    vmovaps %ymm0, 416(%rax)7147; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7148; AVX2-FP-NEXT:    vmovaps %ymm0, 384(%rax)7149; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7150; AVX2-FP-NEXT:    vmovaps %ymm0, 224(%rax)7151; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7152; AVX2-FP-NEXT:    vmovaps %ymm0, 192(%rax)7153; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7154; AVX2-FP-NEXT:    vmovaps %ymm0, 160(%rax)7155; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7156; AVX2-FP-NEXT:    vmovaps %ymm0, 128(%rax)7157; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7158; AVX2-FP-NEXT:    vmovaps %ymm0, 864(%rax)7159; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7160; AVX2-FP-NEXT:    vmovaps %ymm0, 832(%rax)7161; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7162; AVX2-FP-NEXT:    vmovaps %ymm0, 800(%rax)7163; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7164; AVX2-FP-NEXT:    vmovaps %ymm0, 768(%rax)7165; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7166; AVX2-FP-NEXT:    vmovaps %ymm0, 608(%rax)7167; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7168; AVX2-FP-NEXT:    vmovaps %ymm0, 576(%rax)7169; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7170; AVX2-FP-NEXT:    vmovaps %ymm0, 544(%rax)7171; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7172; AVX2-FP-NEXT:    vmovaps %ymm0, 512(%rax)7173; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7174; AVX2-FP-NEXT:    vmovaps %ymm0, 352(%rax)7175; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7176; AVX2-FP-NEXT:    vmovaps %ymm0, 320(%rax)7177; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7178; AVX2-FP-NEXT:    vmovaps %ymm0, 288(%rax)7179; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7180; AVX2-FP-NEXT:    vmovaps %ymm0, 256(%rax)7181; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7182; AVX2-FP-NEXT:    vmovaps %ymm0, 96(%rax)7183; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7184; AVX2-FP-NEXT:    vmovaps %ymm0, 64(%rax)7185; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7186; AVX2-FP-NEXT:    vmovaps %ymm0, 32(%rax)7187; AVX2-FP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7188; AVX2-FP-NEXT:    vmovaps %ymm0, (%rax)7189; AVX2-FP-NEXT:    addq $712, %rsp # imm = 0x2C87190; AVX2-FP-NEXT:    vzeroupper7191; AVX2-FP-NEXT:    retq7192;7193; AVX2-FCP-LABEL: store_i16_stride8_vf64:7194; AVX2-FCP:       # %bb.0:7195; AVX2-FCP-NEXT:    subq $776, %rsp # imm = 0x3087196; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax7197; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r107198; AVX2-FCP-NEXT:    vmovdqa (%rax), %xmm47199; AVX2-FCP-NEXT:    vmovdqa (%r10), %xmm57200; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]7201; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [0,0,0,0,0,0,1,1]7202; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm0, %ymm17203; AVX2-FCP-NEXT:    vmovdqa (%r9), %xmm67204; AVX2-FCP-NEXT:    vmovdqa (%r8), %xmm77205; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]7206; AVX2-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm0 = [0,0,0,1]7207; AVX2-FCP-NEXT:    vpermd %ymm10, %ymm0, %ymm27208; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]7209; AVX2-FCP-NEXT:    vmovdqa (%rcx), %xmm87210; AVX2-FCP-NEXT:    vmovdqa (%rdx), %xmm97211; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]7212; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [0,0,1,1,1,1,0,0]7213; AVX2-FCP-NEXT:    vpermd %ymm11, %ymm0, %ymm127214; AVX2-FCP-NEXT:    vmovdqa (%rsi), %xmm137215; AVX2-FCP-NEXT:    vmovdqa (%rdi), %xmm17216; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm13[0],xmm1[1],xmm13[1],xmm1[2],xmm13[2],xmm1[3],xmm13[3]7217; AVX2-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm14 = [0,1,1,0]7218; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm14, %ymm157219; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm12 = ymm15[0],ymm12[1],ymm15[2,3,4],ymm12[5],ymm15[6,7]7220; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm12[0,1],ymm2[2,3],ymm12[4,5],ymm2[6,7]7221; AVX2-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7222; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm15 = [2,2,2,2,0,0,3,3]7223; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm15, %ymm27224; AVX2-FCP-NEXT:    vpermd %ymm10, %ymm15, %ymm37225; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]7226; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [2,2,3,3,3,3,0,0]7227; AVX2-FCP-NEXT:    vpermd %ymm11, %ymm10, %ymm37228; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm10, %ymm07229; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2,3,4],ymm3[5],ymm0[6,7]7230; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2,3],ymm0[4,5],ymm2[6,7]7231; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7232; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]7233; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm11 = [0,0,0,0,0,0,1,1]7234; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm11, %ymm27235; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]7236; AVX2-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm12 = [0,0,0,1]7237; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm12, %ymm47238; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3],ymm4[4,5,6],ymm2[7]7239; AVX2-FCP-NEXT:    vmovdqa 32(%rax), %xmm107240; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7]7241; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm14 = [0,0,1,1,1,1,0,0]7242; AVX2-FCP-NEXT:    vpermd %ymm8, %ymm14, %ymm57243; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm13[4],xmm1[5],xmm13[5],xmm1[6],xmm13[6],xmm1[7],xmm13[7]7244; AVX2-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm4 = [0,1,1,0]7245; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm4, %ymm67246; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3,4],ymm5[5],ymm6[6,7]7247; AVX2-FCP-NEXT:    vmovdqa 32(%r10), %xmm67248; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm2[2,3],ymm5[4,5],ymm2[6,7]7249; AVX2-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7250; AVX2-FCP-NEXT:    vmovdqa 32(%r9), %xmm57251; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm15, %ymm07252; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm15, %ymm27253; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3],ymm2[4,5,6],ymm0[7]7254; AVX2-FCP-NEXT:    vmovdqa 32(%r8), %xmm77255; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [2,2,3,3,3,3,0,0]7256; AVX2-FCP-NEXT:    vpermd %ymm8, %ymm3, %ymm27257; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm17258; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3,4],ymm2[5],ymm1[6,7]7259; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]7260; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7261; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm10[0],xmm6[1],xmm10[1],xmm6[2],xmm10[2],xmm6[3],xmm10[3]7262; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm11, %ymm17263; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]7264; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm12, %ymm37265; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5,6],ymm1[7]7266; AVX2-FCP-NEXT:    vmovdqa 32(%rcx), %xmm37267; AVX2-FCP-NEXT:    vmovdqa 32(%rdx), %xmm87268; AVX2-FCP-NEXT:    vmovdqa 32(%rsi), %xmm97269; AVX2-FCP-NEXT:    vmovdqa 32(%rdi), %xmm117270; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]7271; AVX2-FCP-NEXT:    vpermd %ymm13, %ymm14, %ymm157272; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]7273; AVX2-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm12 = [0,1,1,0]7274; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm12, %ymm147275; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm14 = ymm14[0],ymm15[1],ymm14[2,3,4],ymm15[5],ymm14[6,7]7276; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm14[0,1],ymm1[2,3],ymm14[4,5],ymm1[6,7]7277; AVX2-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7278; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm15 = [2,2,2,2,0,0,3,3]7279; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm15, %ymm17280; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm15, %ymm27281; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]7282; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm4 = [2,2,3,3,3,3,0,0]7283; AVX2-FCP-NEXT:    vpermd %ymm13, %ymm4, %ymm27284; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm4, %ymm07285; AVX2-FCP-NEXT:    vmovdqa %ymm4, %ymm137286; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3,4],ymm2[5],ymm0[6,7]7287; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]7288; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7289; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm10[4],xmm6[5],xmm10[5],xmm6[6],xmm10[6],xmm6[7],xmm10[7]7290; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]7291; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm7 = [0,0,0,0,0,0,1,1]7292; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm7, %ymm27293; AVX2-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm10 = [0,0,0,1]7294; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm10, %ymm47295; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3],ymm4[4,5,6],ymm2[7]7296; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm8[4],xmm3[4],xmm8[5],xmm3[5],xmm8[6],xmm3[6],xmm8[7],xmm3[7]7297; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7]7298; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm14 = [0,0,1,1,1,1,0,0]7299; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm14, %ymm57300; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm12, %ymm67301; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3,4],ymm5[5],ymm6[6,7]7302; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm2[2,3],ymm5[4,5],ymm2[6,7]7303; AVX2-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7304; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm15, %ymm07305; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm15, %ymm17306; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6],ymm0[7]7307; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm13, %ymm17308; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm13, %ymm27309; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3,4],ymm1[5],ymm2[6,7]7310; AVX2-FCP-NEXT:    vmovdqa 64(%rax), %xmm27311; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]7312; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7313; AVX2-FCP-NEXT:    vmovdqa 64(%r10), %xmm07314; AVX2-FCP-NEXT:    vmovdqa 64(%r9), %xmm17315; AVX2-FCP-NEXT:    vmovdqa 64(%r8), %xmm37316; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]7317; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm7, %ymm57318; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]7319; AVX2-FCP-NEXT:    vpermd %ymm6, %ymm10, %ymm77320; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm7[0,1,2],ymm5[3],ymm7[4,5,6],ymm5[7]7321; AVX2-FCP-NEXT:    vmovdqa 64(%rcx), %xmm77322; AVX2-FCP-NEXT:    vmovdqa 64(%rdx), %xmm87323; AVX2-FCP-NEXT:    vmovdqa 64(%rsi), %xmm97324; AVX2-FCP-NEXT:    vmovdqa 64(%rdi), %xmm107325; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]7326; AVX2-FCP-NEXT:    vpermd %ymm11, %ymm14, %ymm137327; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]7328; AVX2-FCP-NEXT:    vpermd %ymm14, %ymm12, %ymm157329; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm13 = ymm15[0],ymm13[1],ymm15[2,3,4],ymm13[5],ymm15[6,7]7330; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm13[0,1],ymm5[2,3],ymm13[4,5],ymm5[6,7]7331; AVX2-FCP-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7332; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm15 = [2,2,2,2,0,0,3,3]7333; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm15, %ymm47334; AVX2-FCP-NEXT:    vpermd %ymm6, %ymm15, %ymm57335; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3],ymm5[4,5,6],ymm4[7]7336; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm12 = [2,2,3,3,3,3,0,0]7337; AVX2-FCP-NEXT:    vpermd %ymm11, %ymm12, %ymm57338; AVX2-FCP-NEXT:    vpermd %ymm14, %ymm12, %ymm67339; AVX2-FCP-NEXT:    vmovdqa %ymm12, %ymm147340; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3,4],ymm5[5],ymm6[6,7]7341; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1],ymm4[2,3],ymm5[4,5],ymm4[6,7]7342; AVX2-FCP-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7343; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]7344; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]7345; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm11 = [0,0,0,0,0,0,1,1]7346; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm11, %ymm27347; AVX2-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm13 = [0,0,0,1]7348; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm13, %ymm37349; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]7350; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]7351; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]7352; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [0,0,1,1,1,1,0,0]7353; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm10, %ymm57354; AVX2-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm12 = [0,1,1,0]7355; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm12, %ymm67356; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3,4],ymm5[5],ymm6[6,7]7357; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm2[2,3],ymm5[4,5],ymm2[6,7]7358; AVX2-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7359; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm15, %ymm07360; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm15, %ymm17361; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6],ymm0[7]7362; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm14, %ymm17363; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm14, %ymm27364; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3,4],ymm1[5],ymm2[6,7]7365; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]7366; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7367; AVX2-FCP-NEXT:    vmovdqa 96(%rax), %xmm07368; AVX2-FCP-NEXT:    vmovdqa 96(%r10), %xmm17369; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]7370; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm11, %ymm37371; AVX2-FCP-NEXT:    vmovdqa 96(%r9), %xmm47372; AVX2-FCP-NEXT:    vmovdqa 96(%r8), %xmm57373; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]7374; AVX2-FCP-NEXT:    vpermd %ymm6, %ymm13, %ymm77375; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm7[0,1,2],ymm3[3],ymm7[4,5,6],ymm3[7]7376; AVX2-FCP-NEXT:    vmovdqa 96(%rcx), %xmm77377; AVX2-FCP-NEXT:    vmovdqa 96(%rdx), %xmm87378; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]7379; AVX2-FCP-NEXT:    vpermd %ymm9, %ymm10, %ymm107380; AVX2-FCP-NEXT:    vmovdqa 96(%rsi), %xmm117381; AVX2-FCP-NEXT:    vmovdqa 96(%rdi), %xmm137382; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm13[0],xmm11[0],xmm13[1],xmm11[1],xmm13[2],xmm11[2],xmm13[3],xmm11[3]7383; AVX2-FCP-NEXT:    vpermd %ymm14, %ymm12, %ymm157384; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm15[0],ymm10[1],ymm15[2,3,4],ymm10[5],ymm15[6,7]7385; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm10[0,1],ymm3[2,3],ymm10[4,5],ymm3[6,7]7386; AVX2-FCP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7387; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [2,2,2,2,0,0,3,3]7388; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm10, %ymm27389; AVX2-FCP-NEXT:    vpermd %ymm6, %ymm10, %ymm37390; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]7391; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm12 = [2,2,3,3,3,3,0,0]7392; AVX2-FCP-NEXT:    vpermd %ymm9, %ymm12, %ymm37393; AVX2-FCP-NEXT:    vpermd %ymm14, %ymm12, %ymm67394; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0],ymm3[1],ymm6[2,3,4],ymm3[5],ymm6[6,7]7395; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5],ymm2[6,7]7396; AVX2-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7397; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]7398; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]7399; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm2 = [0,0,0,0,0,0,1,1]7400; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm2, %ymm27401; AVX2-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm3 = [0,0,0,1]7402; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm37403; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]7404; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]7405; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm13[4],xmm11[4],xmm13[5],xmm11[5],xmm13[6],xmm11[6],xmm13[7],xmm11[7]7406; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm5 = [0,0,1,1,1,1,0,0]7407; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm5, %ymm57408; AVX2-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm6 = [0,1,1,0]7409; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm6, %ymm67410; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3,4],ymm5[5],ymm6[6,7]7411; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm2[2,3],ymm5[4,5],ymm2[6,7]7412; AVX2-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7413; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm10, %ymm07414; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm10, %ymm17415; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6],ymm0[7]7416; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm12, %ymm17417; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm12, %ymm27418; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3,4],ymm1[5],ymm2[6,7]7419; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]7420; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7421; AVX2-FCP-NEXT:    vmovdqa (%r8), %ymm87422; AVX2-FCP-NEXT:    vmovdqa (%r9), %ymm77423; AVX2-FCP-NEXT:    vmovdqa (%r10), %ymm107424; AVX2-FCP-NEXT:    vmovdqa (%rax), %ymm117425; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm10[0],ymm11[0],ymm10[1],ymm11[1],ymm10[2],ymm11[2],ymm10[3],ymm11[3],ymm10[8],ymm11[8],ymm10[9],ymm11[9],ymm10[10],ymm11[10],ymm10[11],ymm11[11]7426; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [0,0,4,4,4,4,6,5]7427; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm0, %ymm27428; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm12 = ymm8[0],ymm7[0],ymm8[1],ymm7[1],ymm8[2],ymm7[2],ymm8[3],ymm7[3],ymm8[8],ymm7[8],ymm8[9],ymm7[9],ymm8[10],ymm7[10],ymm8[11],ymm7[11]7429; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [0,1,4,5,4,5,5,7]7430; AVX2-FCP-NEXT:    vpermd %ymm12, %ymm0, %ymm37431; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]7432; AVX2-FCP-NEXT:    vmovdqa (%rdi), %ymm137433; AVX2-FCP-NEXT:    vmovdqa (%rsi), %ymm147434; AVX2-FCP-NEXT:    vmovdqa (%rdx), %ymm157435; AVX2-FCP-NEXT:    vmovdqa (%rcx), %ymm27436; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm15[0],ymm2[0],ymm15[1],ymm2[1],ymm15[2],ymm2[2],ymm15[3],ymm2[3],ymm15[8],ymm2[8],ymm15[9],ymm2[9],ymm15[10],ymm2[10],ymm15[11],ymm2[11]7437; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [4,4,2,1,6,5,6,5]7438; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm0, %ymm57439; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm13[0],ymm14[0],ymm13[1],ymm14[1],ymm13[2],ymm14[2],ymm13[3],ymm14[3],ymm13[8],ymm14[8],ymm13[9],ymm14[9],ymm13[10],ymm14[10],ymm13[11],ymm14[11]7440; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [4,5,1,3,5,7,5,7]7441; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm3, %ymm97442; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0],ymm5[1],ymm9[2,3,4],ymm5[5],ymm9[6,7]7443; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm5[0,1],ymm6[2,3],ymm5[4,5],ymm6[6,7]7444; AVX2-FCP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7445; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [0,2,4,6,4,6,6,7]7446; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm3, %ymm47447; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [2,1,6,5,6,5,7,7]7448; AVX2-FCP-NEXT:    vpermd %ymm12, %ymm3, %ymm97449; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm9[0,1,2],ymm4[3],ymm9[4,5,6],ymm4[7]7450; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm5 = [4,6,2,3,6,7,6,7]7451; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm5, %ymm97452; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [6,5,3,3,7,7,7,7]7453; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm3, %ymm07454; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm9[1],ymm0[2,3,4],ymm9[5],ymm0[6,7]7455; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm4[2,3],ymm0[4,5],ymm4[6,7]7456; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7457; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm10[4],ymm11[4],ymm10[5],ymm11[5],ymm10[6],ymm11[6],ymm10[7],ymm11[7],ymm10[12],ymm11[12],ymm10[13],ymm11[13],ymm10[14],ymm11[14],ymm10[15],ymm11[15]7458; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm8[4],ymm7[4],ymm8[5],ymm7[5],ymm8[6],ymm7[6],ymm8[7],ymm7[7],ymm8[12],ymm7[12],ymm8[13],ymm7[13],ymm8[14],ymm7[14],ymm8[15],ymm7[15]7459; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm11 = [0,0,4,4,4,4,6,5]7460; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm11, %ymm67461; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm8 = [0,1,4,5,4,5,5,7]7462; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm8, %ymm77463; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm7[0,1,2],ymm6[3],ymm7[4,5,6],ymm6[7]7464; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm15[4],ymm2[4],ymm15[5],ymm2[5],ymm15[6],ymm2[6],ymm15[7],ymm2[7],ymm15[12],ymm2[12],ymm15[13],ymm2[13],ymm15[14],ymm2[14],ymm15[15],ymm2[15]7465; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm7 = ymm13[4],ymm14[4],ymm13[5],ymm14[5],ymm13[6],ymm14[6],ymm13[7],ymm14[7],ymm13[12],ymm14[12],ymm13[13],ymm14[13],ymm13[14],ymm14[14],ymm13[15],ymm14[15]7466; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm12 = [4,4,2,1,6,5,6,5]7467; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm12, %ymm97468; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [4,5,1,3,5,7,5,7]7469; AVX2-FCP-NEXT:    vpermd %ymm7, %ymm1, %ymm107470; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0],ymm9[1],ymm10[2,3,4],ymm9[5],ymm10[6,7]7471; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm9[0,1],ymm6[2,3],ymm9[4,5],ymm6[6,7]7472; AVX2-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7473; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [0,2,4,6,4,6,6,7]7474; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm1, %ymm07475; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm1 = [2,1,6,5,6,5,7,7]7476; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm1, %ymm47477; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2],ymm0[3],ymm4[4,5,6],ymm0[7]7478; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm5, %ymm27479; AVX2-FCP-NEXT:    vpermd %ymm7, %ymm3, %ymm47480; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2,3,4],ymm2[5],ymm4[6,7]7481; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3],ymm2[4,5],ymm0[6,7]7482; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7483; AVX2-FCP-NEXT:    vmovdqa 32(%r8), %ymm57484; AVX2-FCP-NEXT:    vmovdqa 32(%r9), %ymm37485; AVX2-FCP-NEXT:    vmovdqa 32(%r10), %ymm47486; AVX2-FCP-NEXT:    vmovdqa 32(%rax), %ymm67487; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm4[0],ymm6[0],ymm4[1],ymm6[1],ymm4[2],ymm6[2],ymm4[3],ymm6[3],ymm4[8],ymm6[8],ymm4[9],ymm6[9],ymm4[10],ymm6[10],ymm4[11],ymm6[11]7488; AVX2-FCP-NEXT:    vpermd %ymm7, %ymm11, %ymm97489; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[8],ymm3[8],ymm5[9],ymm3[9],ymm5[10],ymm3[10],ymm5[11],ymm3[11]7490; AVX2-FCP-NEXT:    vpermd %ymm10, %ymm8, %ymm117491; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm9 = ymm11[0,1,2],ymm9[3],ymm11[4,5,6],ymm9[7]7492; AVX2-FCP-NEXT:    vmovdqa 32(%rdi), %ymm117493; AVX2-FCP-NEXT:    vmovdqa 32(%rsi), %ymm137494; AVX2-FCP-NEXT:    vmovdqa 32(%rdx), %ymm157495; AVX2-FCP-NEXT:    vmovdqa 32(%rcx), %ymm27496; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm15[0],ymm2[0],ymm15[1],ymm2[1],ymm15[2],ymm2[2],ymm15[3],ymm2[3],ymm15[8],ymm2[8],ymm15[9],ymm2[9],ymm15[10],ymm2[10],ymm15[11],ymm2[11]7497; AVX2-FCP-NEXT:    vpermd %ymm14, %ymm12, %ymm17498; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm11[0],ymm13[0],ymm11[1],ymm13[1],ymm11[2],ymm13[2],ymm11[3],ymm13[3],ymm11[8],ymm13[8],ymm11[9],ymm13[9],ymm11[10],ymm13[10],ymm11[11],ymm13[11]7499; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm8 = [4,5,1,3,5,7,5,7]7500; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm8, %ymm127501; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0],ymm1[1],ymm12[2,3,4],ymm1[5],ymm12[6,7]7502; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm9[2,3],ymm1[4,5],ymm9[6,7]7503; AVX2-FCP-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7504; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm8 = [0,2,4,6,4,6,6,7]7505; AVX2-FCP-NEXT:    vpermd %ymm7, %ymm8, %ymm17506; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm9 = [2,1,6,5,6,5,7,7]7507; AVX2-FCP-NEXT:    vpermd %ymm10, %ymm9, %ymm77508; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm7[0,1,2],ymm1[3],ymm7[4,5,6],ymm1[7]7509; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [4,6,2,3,6,7,6,7]7510; AVX2-FCP-NEXT:    vpermd %ymm14, %ymm10, %ymm77511; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm12 = [6,5,3,3,7,7,7,7]7512; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm12, %ymm07513; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm7[1],ymm0[2,3,4],ymm7[5],ymm0[6,7]7514; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]7515; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7516; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm4[4],ymm6[4],ymm4[5],ymm6[5],ymm4[6],ymm6[6],ymm4[7],ymm6[7],ymm4[12],ymm6[12],ymm4[13],ymm6[13],ymm4[14],ymm6[14],ymm4[15],ymm6[15]7517; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm5[4],ymm3[4],ymm5[5],ymm3[5],ymm5[6],ymm3[6],ymm5[7],ymm3[7],ymm5[12],ymm3[12],ymm5[13],ymm3[13],ymm5[14],ymm3[14],ymm5[15],ymm3[15]7518; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm7 = [0,0,4,4,4,4,6,5]7519; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm7, %ymm37520; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm14 = [0,1,4,5,4,5,5,7]7521; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm14, %ymm47522; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3],ymm4[4,5,6],ymm3[7]7523; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm15[4],ymm2[4],ymm15[5],ymm2[5],ymm15[6],ymm2[6],ymm15[7],ymm2[7],ymm15[12],ymm2[12],ymm15[13],ymm2[13],ymm15[14],ymm2[14],ymm15[15],ymm2[15]7524; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm11[4],ymm13[4],ymm11[5],ymm13[5],ymm11[6],ymm13[6],ymm11[7],ymm13[7],ymm11[12],ymm13[12],ymm11[13],ymm13[13],ymm11[14],ymm13[14],ymm11[15],ymm13[15]7525; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm13 = [4,4,2,1,6,5,6,5]7526; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm13, %ymm57527; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm6 = [4,5,1,3,5,7,5,7]7528; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm6, %ymm67529; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3,4],ymm5[5],ymm6[6,7]7530; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm5[0,1],ymm3[2,3],ymm5[4,5],ymm3[6,7]7531; AVX2-FCP-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7532; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm8, %ymm07533; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm9, %ymm17534; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6],ymm0[7]7535; AVX2-FCP-NEXT:    vpermd %ymm2, %ymm10, %ymm17536; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm12, %ymm27537; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3,4],ymm1[5],ymm2[6,7]7538; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]7539; AVX2-FCP-NEXT:    vmovdqu %ymm0, (%rsp) # 32-byte Spill7540; AVX2-FCP-NEXT:    vmovdqa 64(%r8), %ymm97541; AVX2-FCP-NEXT:    vmovdqa 64(%r9), %ymm87542; AVX2-FCP-NEXT:    vmovdqa 64(%r10), %ymm37543; AVX2-FCP-NEXT:    vmovdqa 64(%rax), %ymm47544; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11]7545; AVX2-FCP-NEXT:    vpermd %ymm5, %ymm7, %ymm67546; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]7547; AVX2-FCP-NEXT:    vpermd %ymm7, %ymm14, %ymm107548; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm10[0,1,2],ymm6[3],ymm10[4,5,6],ymm6[7]7549; AVX2-FCP-NEXT:    vmovdqa 64(%rdi), %ymm127550; AVX2-FCP-NEXT:    vmovdqa 64(%rsi), %ymm157551; AVX2-FCP-NEXT:    vmovdqa 64(%rdx), %ymm117552; AVX2-FCP-NEXT:    vmovdqa 64(%rcx), %ymm107553; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm11[0],ymm10[0],ymm11[1],ymm10[1],ymm11[2],ymm10[2],ymm11[3],ymm10[3],ymm11[8],ymm10[8],ymm11[9],ymm10[9],ymm11[10],ymm10[10],ymm11[11],ymm10[11]7554; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm13, %ymm27555; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm12[0],ymm15[0],ymm12[1],ymm15[1],ymm12[2],ymm15[2],ymm12[3],ymm15[3],ymm12[8],ymm15[8],ymm12[9],ymm15[9],ymm12[10],ymm15[10],ymm12[11],ymm15[11]7556; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm13 = [4,5,1,3,5,7,5,7]7557; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm13, %ymm147558; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm14[0],ymm2[1],ymm14[2,3,4],ymm2[5],ymm14[6,7]7559; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm6[2,3],ymm2[4,5],ymm6[6,7]7560; AVX2-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7561; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm6 = [0,2,4,6,4,6,6,7]7562; AVX2-FCP-NEXT:    vpermd %ymm5, %ymm6, %ymm27563; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm14 = [2,1,6,5,6,5,7,7]7564; AVX2-FCP-NEXT:    vpermd %ymm7, %ymm14, %ymm57565; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2],ymm2[3],ymm5[4,5,6],ymm2[7]7566; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm5 = [4,6,2,3,6,7,6,7]7567; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm5, %ymm17568; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm7 = [6,5,3,3,7,7,7,7]7569; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm7, %ymm07570; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4],ymm1[5],ymm0[6,7]7571; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2,3],ymm0[4,5],ymm2[6,7]7572; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7573; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm3[4],ymm4[4],ymm3[5],ymm4[5],ymm3[6],ymm4[6],ymm3[7],ymm4[7],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15]7574; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15]7575; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm9 = [0,0,4,4,4,4,6,5]7576; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm9, %ymm27577; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [0,1,4,5,4,5,5,7]7578; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm37579; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]7580; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm11[4],ymm10[4],ymm11[5],ymm10[5],ymm11[6],ymm10[6],ymm11[7],ymm10[7],ymm11[12],ymm10[12],ymm11[13],ymm10[13],ymm11[14],ymm10[14],ymm11[15],ymm10[15]7581; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm12[4],ymm15[4],ymm12[5],ymm15[5],ymm12[6],ymm15[6],ymm12[7],ymm15[7],ymm12[12],ymm15[12],ymm12[13],ymm15[13],ymm12[14],ymm15[14],ymm12[15],ymm15[15]7582; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm7 = [4,4,2,1,6,5,6,5]7583; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm7, %ymm77584; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm13, %ymm87585; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm8[0],ymm7[1],ymm8[2,3,4],ymm7[5],ymm8[6,7]7586; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1],ymm2[2,3],ymm7[4,5],ymm2[6,7]7587; AVX2-FCP-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7588; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm6, %ymm07589; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm14, %ymm17590; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5,6],ymm0[7]7591; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm5, %ymm17592; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm2 = [6,5,3,3,7,7,7,7]7593; AVX2-FCP-NEXT:    vpermd %ymm4, %ymm2, %ymm27594; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3,4],ymm1[5],ymm2[6,7]7595; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]7596; AVX2-FCP-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill7597; AVX2-FCP-NEXT:    vmovdqa 96(%r8), %ymm67598; AVX2-FCP-NEXT:    vmovdqa 96(%r9), %ymm57599; AVX2-FCP-NEXT:    vmovdqa 96(%r10), %ymm37600; AVX2-FCP-NEXT:    vmovdqa 96(%rax), %ymm47601; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm8 = ymm3[0],ymm4[0],ymm3[1],ymm4[1],ymm3[2],ymm4[2],ymm3[3],ymm4[3],ymm3[8],ymm4[8],ymm3[9],ymm4[9],ymm3[10],ymm4[10],ymm3[11],ymm4[11]7602; AVX2-FCP-NEXT:    vpermd %ymm8, %ymm9, %ymm97603; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm6[0],ymm5[0],ymm6[1],ymm5[1],ymm6[2],ymm5[2],ymm6[3],ymm5[3],ymm6[8],ymm5[8],ymm6[9],ymm5[9],ymm6[10],ymm5[10],ymm6[11],ymm5[11]7604; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [0,1,4,5,4,5,5,7]7605; AVX2-FCP-NEXT:    vpermd %ymm10, %ymm0, %ymm117606; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm9 = ymm11[0,1,2],ymm9[3],ymm11[4,5,6],ymm9[7]7607; AVX2-FCP-NEXT:    vmovdqa 96(%rdi), %ymm117608; AVX2-FCP-NEXT:    vmovdqa 96(%rsi), %ymm127609; AVX2-FCP-NEXT:    vmovdqa 96(%rdx), %ymm147610; AVX2-FCP-NEXT:    vmovdqa 96(%rcx), %ymm157611; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm14[0],ymm15[0],ymm14[1],ymm15[1],ymm14[2],ymm15[2],ymm14[3],ymm15[3],ymm14[8],ymm15[8],ymm14[9],ymm15[9],ymm14[10],ymm15[10],ymm14[11],ymm15[11]7612; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm0 = [4,4,2,1,6,5,6,5]7613; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm0, %ymm27614; AVX2-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm11[0],ymm12[0],ymm11[1],ymm12[1],ymm11[2],ymm12[2],ymm11[3],ymm12[3],ymm11[8],ymm12[8],ymm11[9],ymm12[9],ymm11[10],ymm12[10],ymm11[11],ymm12[11]7615; AVX2-FCP-NEXT:    vmovdqa %ymm13, %ymm77616; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm13, %ymm137617; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm13[0],ymm2[1],ymm13[2,3,4],ymm2[5],ymm13[6,7]7618; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm9[2,3],ymm2[4,5],ymm9[6,7]7619; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm9 = [0,2,4,6,4,6,6,7]7620; AVX2-FCP-NEXT:    vpermd %ymm8, %ymm9, %ymm87621; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm13 = [2,1,6,5,6,5,7,7]7622; AVX2-FCP-NEXT:    vpermd %ymm10, %ymm13, %ymm97623; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2],ymm8[3],ymm9[4,5,6],ymm8[7]7624; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm9 = [4,6,2,3,6,7,6,7]7625; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm9, %ymm17626; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm10 = [6,5,3,3,7,7,7,7]7627; AVX2-FCP-NEXT:    vpermd %ymm0, %ymm10, %ymm07628; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4],ymm1[5],ymm0[6,7]7629; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm8[2,3],ymm0[4,5],ymm8[6,7]7630; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm3[4],ymm4[4],ymm3[5],ymm4[5],ymm3[6],ymm4[6],ymm3[7],ymm4[7],ymm3[12],ymm4[12],ymm3[13],ymm4[13],ymm3[14],ymm4[14],ymm3[15],ymm4[15]7631; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]7632; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm4 = [0,0,4,4,4,4,6,5]7633; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm4, %ymm47634; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm5 = [0,1,4,5,4,5,5,7]7635; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm5, %ymm57636; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3],ymm5[4,5,6],ymm4[7]7637; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm14[4],ymm15[4],ymm14[5],ymm15[5],ymm14[6],ymm15[6],ymm14[7],ymm15[7],ymm14[12],ymm15[12],ymm14[13],ymm15[13],ymm14[14],ymm15[14],ymm14[15],ymm15[15]7638; AVX2-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm11[4],ymm12[4],ymm11[5],ymm12[5],ymm11[6],ymm12[6],ymm11[7],ymm12[7],ymm11[12],ymm12[12],ymm11[13],ymm12[13],ymm11[14],ymm12[14],ymm11[15],ymm12[15]7639; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm8 = [4,4,2,1,6,5,6,5]7640; AVX2-FCP-NEXT:    vpermd %ymm5, %ymm8, %ymm87641; AVX2-FCP-NEXT:    vpermd %ymm6, %ymm7, %ymm97642; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0],ymm8[1],ymm9[2,3,4],ymm8[5],ymm9[6,7]7643; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1],ymm4[2,3],ymm8[4,5],ymm4[6,7]7644; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm7 = [0,2,4,6,4,6,6,7]7645; AVX2-FCP-NEXT:    vpermd %ymm1, %ymm7, %ymm17646; AVX2-FCP-NEXT:    vpermd %ymm3, %ymm13, %ymm37647; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5,6],ymm1[7]7648; AVX2-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [4,6,2,3,6,7,6,7]7649; AVX2-FCP-NEXT:    vpermd %ymm5, %ymm3, %ymm37650; AVX2-FCP-NEXT:    vpermd %ymm6, %ymm10, %ymm57651; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm5[0],ymm3[1],ymm5[2,3,4],ymm3[5],ymm5[6,7]7652; AVX2-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,3],ymm3[4,5],ymm1[6,7]7653; AVX2-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax7654; AVX2-FCP-NEXT:    vmovdqa %ymm1, 992(%rax)7655; AVX2-FCP-NEXT:    vmovdqa %ymm4, 960(%rax)7656; AVX2-FCP-NEXT:    vmovdqa %ymm0, 928(%rax)7657; AVX2-FCP-NEXT:    vmovdqa %ymm2, 896(%rax)7658; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7659; AVX2-FCP-NEXT:    vmovaps %ymm0, 736(%rax)7660; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7661; AVX2-FCP-NEXT:    vmovaps %ymm0, 704(%rax)7662; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7663; AVX2-FCP-NEXT:    vmovaps %ymm0, 672(%rax)7664; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7665; AVX2-FCP-NEXT:    vmovaps %ymm0, 640(%rax)7666; AVX2-FCP-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload7667; AVX2-FCP-NEXT:    vmovaps %ymm0, 480(%rax)7668; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7669; AVX2-FCP-NEXT:    vmovaps %ymm0, 448(%rax)7670; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7671; AVX2-FCP-NEXT:    vmovaps %ymm0, 416(%rax)7672; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7673; AVX2-FCP-NEXT:    vmovaps %ymm0, 384(%rax)7674; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7675; AVX2-FCP-NEXT:    vmovaps %ymm0, 224(%rax)7676; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7677; AVX2-FCP-NEXT:    vmovaps %ymm0, 192(%rax)7678; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7679; AVX2-FCP-NEXT:    vmovaps %ymm0, 160(%rax)7680; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7681; AVX2-FCP-NEXT:    vmovaps %ymm0, 128(%rax)7682; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7683; AVX2-FCP-NEXT:    vmovaps %ymm0, 864(%rax)7684; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7685; AVX2-FCP-NEXT:    vmovaps %ymm0, 832(%rax)7686; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7687; AVX2-FCP-NEXT:    vmovaps %ymm0, 800(%rax)7688; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7689; AVX2-FCP-NEXT:    vmovaps %ymm0, 768(%rax)7690; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7691; AVX2-FCP-NEXT:    vmovaps %ymm0, 608(%rax)7692; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7693; AVX2-FCP-NEXT:    vmovaps %ymm0, 576(%rax)7694; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7695; AVX2-FCP-NEXT:    vmovaps %ymm0, 544(%rax)7696; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7697; AVX2-FCP-NEXT:    vmovaps %ymm0, 512(%rax)7698; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7699; AVX2-FCP-NEXT:    vmovaps %ymm0, 352(%rax)7700; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7701; AVX2-FCP-NEXT:    vmovaps %ymm0, 320(%rax)7702; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7703; AVX2-FCP-NEXT:    vmovaps %ymm0, 288(%rax)7704; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7705; AVX2-FCP-NEXT:    vmovaps %ymm0, 256(%rax)7706; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7707; AVX2-FCP-NEXT:    vmovaps %ymm0, 96(%rax)7708; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7709; AVX2-FCP-NEXT:    vmovaps %ymm0, 64(%rax)7710; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7711; AVX2-FCP-NEXT:    vmovaps %ymm0, 32(%rax)7712; AVX2-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload7713; AVX2-FCP-NEXT:    vmovaps %ymm0, (%rax)7714; AVX2-FCP-NEXT:    addq $776, %rsp # imm = 0x3087715; AVX2-FCP-NEXT:    vzeroupper7716; AVX2-FCP-NEXT:    retq7717;7718; AVX512-LABEL: store_i16_stride8_vf64:7719; AVX512:       # %bb.0:7720; AVX512-NEXT:    subq $392, %rsp # imm = 0x1887721; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax7722; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r107723; AVX512-NEXT:    vmovdqa (%r10), %xmm17724; AVX512-NEXT:    vmovdqa (%rax), %xmm27725; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]7726; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm207727; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm217728; AVX512-NEXT:    vmovdqa (%r9), %xmm27729; AVX512-NEXT:    vmovdqa (%r8), %xmm37730; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]7731; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm227732; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm237733; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]7734; AVX512-NEXT:    vpermt2d %zmm0, %zmm16, %zmm17735; AVX512-NEXT:    vmovdqa (%rcx), %xmm27736; AVX512-NEXT:    vmovdqa (%rdx), %xmm37737; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]7738; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm247739; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm257740; AVX512-NEXT:    vmovdqa (%rsi), %xmm37741; AVX512-NEXT:    vmovdqa (%rdi), %xmm47742; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]7743; AVX512-NEXT:    vmovdqa64 %xmm4, %xmm267744; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm277745; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]7746; AVX512-NEXT:    vpermt2d %zmm0, %zmm17, %zmm27747; AVX512-NEXT:    movb $-86, %r11b7748; AVX512-NEXT:    kmovw %r11d, %k17749; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}7750; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7751; AVX512-NEXT:    vmovdqa 96(%r10), %ymm07752; AVX512-NEXT:    vmovdqa 96(%rax), %ymm17753; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]7754; AVX512-NEXT:    vmovdqa 96(%r9), %ymm37755; AVX512-NEXT:    vmovdqa 96(%r8), %ymm47756; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]7757; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]7758; AVX512-NEXT:    vpermt2d %zmm2, %zmm18, %zmm57759; AVX512-NEXT:    vmovdqa 96(%rcx), %ymm27760; AVX512-NEXT:    vmovdqa 96(%rdx), %ymm67761; AVX512-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]7762; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm6[0],ymm2[0],ymm6[1],ymm2[1],ymm6[2],ymm2[2],ymm6[3],ymm2[3],ymm6[8],ymm2[8],ymm6[9],ymm2[9],ymm6[10],ymm2[10],ymm6[11],ymm2[11]7763; AVX512-NEXT:    vmovdqa 96(%rsi), %ymm127764; AVX512-NEXT:    vmovdqa 96(%rdi), %ymm137765; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]7766; AVX512-NEXT:    vpermt2d %zmm10, %zmm19, %zmm77767; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}7768; AVX512-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7769; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]7770; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]7771; AVX512-NEXT:    vpermt2d %zmm0, %zmm18, %zmm17772; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm6[4],ymm2[4],ymm6[5],ymm2[5],ymm6[6],ymm2[6],ymm6[7],ymm2[7],ymm6[12],ymm2[12],ymm6[13],ymm2[13],ymm6[14],ymm2[14],ymm6[15],ymm2[15]7773; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm12[4],ymm13[5],ymm12[5],ymm13[6],ymm12[6],ymm13[7],ymm12[7],ymm13[12],ymm12[12],ymm13[13],ymm12[13],ymm13[14],ymm12[14],ymm13[15],ymm12[15]7774; AVX512-NEXT:    vpermt2d %zmm0, %zmm19, %zmm27775; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}7776; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7777; AVX512-NEXT:    vmovdqa 96(%r10), %xmm07778; AVX512-NEXT:    vmovdqa 96(%rax), %xmm17779; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]7780; AVX512-NEXT:    vmovdqa 96(%r9), %xmm37781; AVX512-NEXT:    vmovdqa 96(%r8), %xmm47782; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]7783; AVX512-NEXT:    vpermt2d %zmm2, %zmm16, %zmm57784; AVX512-NEXT:    vmovdqa 96(%rcx), %xmm27785; AVX512-NEXT:    vmovdqa 96(%rdx), %xmm67786; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]7787; AVX512-NEXT:    vmovdqa 96(%rsi), %xmm137788; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm147789; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]7790; AVX512-NEXT:    vpermt2d %zmm10, %zmm17, %zmm77791; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}7792; AVX512-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7793; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]7794; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]7795; AVX512-NEXT:    vpermt2d %zmm0, %zmm16, %zmm17796; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]7797; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]7798; AVX512-NEXT:    vpermt2d %zmm0, %zmm17, %zmm27799; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}7800; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7801; AVX512-NEXT:    vmovdqa 64(%r10), %ymm67802; AVX512-NEXT:    vmovdqa 64(%rax), %ymm57803; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11]7804; AVX512-NEXT:    vmovdqa 64(%r9), %ymm37805; AVX512-NEXT:    vmovdqa 64(%r8), %ymm47806; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]7807; AVX512-NEXT:    vpermt2d %zmm2, %zmm18, %zmm107808; AVX512-NEXT:    vmovdqa 64(%rcx), %ymm27809; AVX512-NEXT:    vmovdqa 64(%rdx), %ymm137810; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm13[0],ymm2[0],ymm13[1],ymm2[1],ymm13[2],ymm2[2],ymm13[3],ymm2[3],ymm13[8],ymm2[8],ymm13[9],ymm2[9],ymm13[10],ymm2[10],ymm13[11],ymm2[11]7811; AVX512-NEXT:    vmovdqa 64(%rsi), %ymm17812; AVX512-NEXT:    vmovdqa 64(%rdi), %ymm07813; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]7814; AVX512-NEXT:    vpermt2d %zmm14, %zmm19, %zmm77815; AVX512-NEXT:    vmovdqa64 %zmm10, %zmm7 {%k1}7816; AVX512-NEXT:    vmovdqu64 %zmm7, (%rsp) # 64-byte Spill7817; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15]7818; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]7819; AVX512-NEXT:    vpermt2d %zmm5, %zmm18, %zmm37820; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm2[4],ymm13[5],ymm2[5],ymm13[6],ymm2[6],ymm13[7],ymm2[7],ymm13[12],ymm2[12],ymm13[13],ymm2[13],ymm13[14],ymm2[14],ymm13[15],ymm2[15]7821; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]7822; AVX512-NEXT:    vpermt2d %zmm2, %zmm19, %zmm07823; AVX512-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k1}7824; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7825; AVX512-NEXT:    vmovdqa 64(%r10), %xmm77826; AVX512-NEXT:    vmovdqa 64(%rax), %xmm27827; AVX512-NEXT:    vmovdqa 64(%r9), %xmm87828; AVX512-NEXT:    vmovdqa 64(%r8), %xmm37829; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm7[0],xmm2[1],xmm7[1],xmm2[2],xmm7[2],xmm2[3],xmm7[3]7830; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3]7831; AVX512-NEXT:    vpermt2d %zmm4, %zmm16, %zmm57832; AVX512-NEXT:    vmovdqa 64(%rcx), %xmm47833; AVX512-NEXT:    vmovdqa 64(%rdx), %xmm67834; AVX512-NEXT:    vmovdqa 64(%rsi), %xmm147835; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm07836; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]7837; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]7838; AVX512-NEXT:    vpermt2d %zmm1, %zmm17, %zmm97839; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm9 {%k1}7840; AVX512-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7841; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm7[4],xmm2[5],xmm7[5],xmm2[6],xmm7[6],xmm2[7],xmm7[7]7842; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm8[4],xmm3[5],xmm8[5],xmm3[6],xmm8[6],xmm3[7],xmm8[7]7843; AVX512-NEXT:    vpermt2d %zmm1, %zmm16, %zmm27844; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]7845; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]7846; AVX512-NEXT:    vpermt2d %zmm1, %zmm17, %zmm157847; AVX512-NEXT:    vmovdqa64 %zmm2, %zmm15 {%k1}7848; AVX512-NEXT:    vmovdqa 32(%r10), %ymm87849; AVX512-NEXT:    vmovdqa 32(%rax), %ymm47850; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm4[0],ymm8[0],ymm4[1],ymm8[1],ymm4[2],ymm8[2],ymm4[3],ymm8[3],ymm4[8],ymm8[8],ymm4[9],ymm8[9],ymm4[10],ymm8[10],ymm4[11],ymm8[11]7851; AVX512-NEXT:    vmovdqa 32(%r9), %ymm37852; AVX512-NEXT:    vmovdqa 32(%r8), %ymm57853; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[8],ymm3[8],ymm5[9],ymm3[9],ymm5[10],ymm3[10],ymm5[11],ymm3[11]7854; AVX512-NEXT:    vpermt2d %zmm2, %zmm18, %zmm67855; AVX512-NEXT:    vmovdqa 32(%rcx), %ymm27856; AVX512-NEXT:    vmovdqa 32(%rdx), %ymm77857; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm7[0],ymm2[0],ymm7[1],ymm2[1],ymm7[2],ymm2[2],ymm7[3],ymm2[3],ymm7[8],ymm2[8],ymm7[9],ymm2[9],ymm7[10],ymm2[10],ymm7[11],ymm2[11]7858; AVX512-NEXT:    vmovdqa 32(%rsi), %ymm17859; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm07860; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]7861; AVX512-NEXT:    vpermt2d %zmm9, %zmm19, %zmm147862; AVX512-NEXT:    vmovdqa64 %zmm6, %zmm14 {%k1}7863; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm4[4],ymm8[4],ymm4[5],ymm8[5],ymm4[6],ymm8[6],ymm4[7],ymm8[7],ymm4[12],ymm8[12],ymm4[13],ymm8[13],ymm4[14],ymm8[14],ymm4[15],ymm8[15]7864; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm3[4],ymm5[5],ymm3[5],ymm5[6],ymm3[6],ymm5[7],ymm3[7],ymm5[12],ymm3[12],ymm5[13],ymm3[13],ymm5[14],ymm3[14],ymm5[15],ymm3[15]7865; AVX512-NEXT:    vpermt2d %zmm4, %zmm18, %zmm57866; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm7[4],ymm2[4],ymm7[5],ymm2[5],ymm7[6],ymm2[6],ymm7[7],ymm2[7],ymm7[12],ymm2[12],ymm7[13],ymm2[13],ymm7[14],ymm2[14],ymm7[15],ymm2[15]7867; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm13 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]7868; AVX512-NEXT:    vpermt2d %zmm2, %zmm19, %zmm137869; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm13 {%k1}7870; AVX512-NEXT:    vmovdqa 32(%r10), %xmm107871; AVX512-NEXT:    vmovdqa 32(%rax), %xmm37872; AVX512-NEXT:    vmovdqa 32(%r9), %xmm47873; AVX512-NEXT:    vmovdqa 32(%r8), %xmm57874; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]7875; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]7876; AVX512-NEXT:    vpermt2d %zmm2, %zmm16, %zmm67877; AVX512-NEXT:    vmovdqa 32(%rcx), %xmm77878; AVX512-NEXT:    vmovdqa 32(%rdx), %xmm87879; AVX512-NEXT:    vmovdqa 32(%rsi), %xmm97880; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm07881; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]7882; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3]7883; AVX512-NEXT:    vpermt2d %zmm1, %zmm17, %zmm127884; AVX512-NEXT:    vmovdqa64 %zmm6, %zmm12 {%k1}7885; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]7886; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]7887; AVX512-NEXT:    vpermt2d %zmm1, %zmm16, %zmm37888; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]7889; AVX512-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]7890; AVX512-NEXT:    vpermt2d %zmm1, %zmm17, %zmm47891; AVX512-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}7892; AVX512-NEXT:    vmovdqa (%r10), %ymm107893; AVX512-NEXT:    vmovdqa (%rax), %ymm37894; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm3[0],ymm10[0],ymm3[1],ymm10[1],ymm3[2],ymm10[2],ymm3[3],ymm10[3],ymm3[8],ymm10[8],ymm3[9],ymm10[9],ymm3[10],ymm10[10],ymm3[11],ymm10[11]7895; AVX512-NEXT:    vmovdqa (%r9), %ymm57896; AVX512-NEXT:    vmovdqa (%r8), %ymm67897; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm6[0],ymm5[0],ymm6[1],ymm5[1],ymm6[2],ymm5[2],ymm6[3],ymm5[3],ymm6[8],ymm5[8],ymm6[9],ymm5[9],ymm6[10],ymm5[10],ymm6[11],ymm5[11]7898; AVX512-NEXT:    vpermt2d %zmm0, %zmm18, %zmm77899; AVX512-NEXT:    vmovdqa (%rcx), %ymm87900; AVX512-NEXT:    vmovdqa (%rdx), %ymm97901; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]7902; AVX512-NEXT:    vmovdqa (%rsi), %ymm27903; AVX512-NEXT:    vmovdqa (%rdi), %ymm17904; AVX512-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]7905; AVX512-NEXT:    vpermt2d %zmm11, %zmm19, %zmm07906; AVX512-NEXT:    vmovdqa64 %zmm7, %zmm0 {%k1}7907; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm10[4],ymm3[5],ymm10[5],ymm3[6],ymm10[6],ymm3[7],ymm10[7],ymm3[12],ymm10[12],ymm3[13],ymm10[13],ymm3[14],ymm10[14],ymm3[15],ymm10[15]7908; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]7909; AVX512-NEXT:    vpermt2d %zmm3, %zmm18, %zmm57910; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15]7911; AVX512-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]7912; AVX512-NEXT:    vpermt2d %zmm3, %zmm19, %zmm17913; AVX512-NEXT:    vmovdqa64 %zmm5, %zmm1 {%k1}7914; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm27915; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm37916; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]7917; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm37918; AVX512-NEXT:    vmovdqa64 %xmm23, %xmm57919; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]7920; AVX512-NEXT:    vpermt2d %zmm2, %zmm16, %zmm37921; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm27922; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm57923; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]7924; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm57925; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm67926; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]7927; AVX512-NEXT:    vpermt2d %zmm2, %zmm17, %zmm57928; AVX512-NEXT:    vmovdqa64 %zmm3, %zmm5 {%k1}7929; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax7930; AVX512-NEXT:    vmovdqa64 %zmm5, (%rax)7931; AVX512-NEXT:    vmovdqa64 %zmm1, 192(%rax)7932; AVX512-NEXT:    vmovdqa64 %zmm0, 128(%rax)7933; AVX512-NEXT:    vmovdqa64 %zmm4, 320(%rax)7934; AVX512-NEXT:    vmovdqa64 %zmm12, 256(%rax)7935; AVX512-NEXT:    vmovdqa64 %zmm13, 448(%rax)7936; AVX512-NEXT:    vmovdqa64 %zmm14, 384(%rax)7937; AVX512-NEXT:    vmovdqa64 %zmm15, 576(%rax)7938; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload7939; AVX512-NEXT:    vmovaps %zmm0, 512(%rax)7940; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload7941; AVX512-NEXT:    vmovaps %zmm0, 704(%rax)7942; AVX512-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload7943; AVX512-NEXT:    vmovaps %zmm0, 640(%rax)7944; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload7945; AVX512-NEXT:    vmovaps %zmm0, 832(%rax)7946; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload7947; AVX512-NEXT:    vmovaps %zmm0, 768(%rax)7948; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload7949; AVX512-NEXT:    vmovaps %zmm0, 960(%rax)7950; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload7951; AVX512-NEXT:    vmovaps %zmm0, 896(%rax)7952; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload7953; AVX512-NEXT:    vmovaps %zmm0, 64(%rax)7954; AVX512-NEXT:    addq $392, %rsp # imm = 0x1887955; AVX512-NEXT:    vzeroupper7956; AVX512-NEXT:    retq7957;7958; AVX512-FCP-LABEL: store_i16_stride8_vf64:7959; AVX512-FCP:       # %bb.0:7960; AVX512-FCP-NEXT:    subq $328, %rsp # imm = 0x1487961; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax7962; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r107963; AVX512-FCP-NEXT:    vmovdqa (%r10), %xmm17964; AVX512-FCP-NEXT:    vmovdqa (%rax), %xmm27965; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]7966; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm217967; AVX512-FCP-NEXT:    vmovdqa64 %xmm1, %xmm227968; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm27969; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm37970; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]7971; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm237972; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm247973; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]7974; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm16, %zmm17975; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm27976; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm37977; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]7978; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm257979; AVX512-FCP-NEXT:    vmovdqa64 %xmm2, %xmm267980; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm37981; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm47982; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]7983; AVX512-FCP-NEXT:    vmovdqa64 %xmm4, %xmm277984; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm287985; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [0,16,1,0,1,17,0,0,2,18,3,0,3,19,0,0]7986; AVX512-FCP-NEXT:    vpermi2d %zmm0, %zmm2, %zmm207987; AVX512-FCP-NEXT:    movb $-86, %r11b7988; AVX512-FCP-NEXT:    kmovw %r11d, %k17989; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, %zmm20 {%k1}7990; AVX512-FCP-NEXT:    vmovdqa 96(%r10), %ymm07991; AVX512-FCP-NEXT:    vmovdqa 96(%rax), %ymm17992; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]7993; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %ymm37994; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %ymm47995; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]7996; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]7997; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm57998; AVX512-FCP-NEXT:    vmovdqa 96(%rcx), %ymm27999; AVX512-FCP-NEXT:    vmovdqa 96(%rdx), %ymm68000; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm6[0],ymm2[0],ymm6[1],ymm2[1],ymm6[2],ymm2[2],ymm6[3],ymm2[3],ymm6[8],ymm2[8],ymm6[9],ymm2[9],ymm6[10],ymm2[10],ymm6[11],ymm2[11]8001; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]8002; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %ymm128003; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %ymm138004; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]8005; AVX512-FCP-NEXT:    vpermt2d %zmm11, %zmm19, %zmm78006; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}8007; AVX512-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8008; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]8009; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]8010; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm17, %zmm18011; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm6[4],ymm2[4],ymm6[5],ymm2[5],ymm6[6],ymm2[6],ymm6[7],ymm2[7],ymm6[12],ymm2[12],ymm6[13],ymm2[13],ymm6[14],ymm2[14],ymm6[15],ymm2[15]8012; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm12[4],ymm13[5],ymm12[5],ymm13[6],ymm12[6],ymm13[7],ymm12[7],ymm13[12],ymm12[12],ymm13[13],ymm12[13],ymm13[14],ymm12[14],ymm13[15],ymm12[15]8013; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm19, %zmm28014; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}8015; AVX512-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8016; AVX512-FCP-NEXT:    vmovdqa 96(%r10), %xmm08017; AVX512-FCP-NEXT:    vmovdqa 96(%rax), %xmm18018; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]8019; AVX512-FCP-NEXT:    vmovdqa 96(%r9), %xmm38020; AVX512-FCP-NEXT:    vmovdqa 96(%r8), %xmm48021; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]8022; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm16, %zmm58023; AVX512-FCP-NEXT:    vmovdqa 96(%rcx), %xmm28024; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]8025; AVX512-FCP-NEXT:    vmovdqa 96(%rdx), %xmm68026; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]8027; AVX512-FCP-NEXT:    vmovdqa 96(%rsi), %xmm138028; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm158029; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3]8030; AVX512-FCP-NEXT:    vpermt2d %zmm12, %zmm18, %zmm78031; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}8032; AVX512-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8033; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]8034; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]8035; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm16, %zmm18036; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]8037; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm15[4],xmm13[4],xmm15[5],xmm13[5],xmm15[6],xmm13[6],xmm15[7],xmm13[7]8038; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm18, %zmm28039; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}8040; AVX512-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8041; AVX512-FCP-NEXT:    vmovdqa 64(%r10), %ymm68042; AVX512-FCP-NEXT:    vmovdqa 64(%rax), %ymm58043; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11]8044; AVX512-FCP-NEXT:    vmovdqa 64(%r9), %ymm38045; AVX512-FCP-NEXT:    vmovdqa 64(%r8), %ymm48046; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm12 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]8047; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm128048; AVX512-FCP-NEXT:    vmovdqa 64(%rcx), %ymm28049; AVX512-FCP-NEXT:    vmovdqa 64(%rdx), %ymm138050; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm13[0],ymm2[0],ymm13[1],ymm2[1],ymm13[2],ymm2[2],ymm13[3],ymm2[3],ymm13[8],ymm2[8],ymm13[9],ymm2[9],ymm13[10],ymm2[10],ymm13[11],ymm2[11]8051; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %ymm18052; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %ymm08053; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]8054; AVX512-FCP-NEXT:    vpermt2d %zmm15, %zmm19, %zmm78055; AVX512-FCP-NEXT:    vmovdqa64 %zmm12, %zmm7 {%k1}8056; AVX512-FCP-NEXT:    vmovdqu64 %zmm7, (%rsp) # 64-byte Spill8057; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15]8058; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]8059; AVX512-FCP-NEXT:    vpermt2d %zmm5, %zmm17, %zmm38060; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm2[4],ymm13[5],ymm2[5],ymm13[6],ymm2[6],ymm13[7],ymm2[7],ymm13[12],ymm2[12],ymm13[13],ymm2[13],ymm13[14],ymm2[14],ymm13[15],ymm2[15]8061; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]8062; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm19, %zmm08063; AVX512-FCP-NEXT:    vmovdqa 64(%r10), %xmm88064; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k1}8065; AVX512-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8066; AVX512-FCP-NEXT:    vmovdqa 64(%rax), %xmm28067; AVX512-FCP-NEXT:    vmovdqa 64(%r9), %xmm78068; AVX512-FCP-NEXT:    vmovdqa 64(%r8), %xmm38069; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm8[0],xmm2[1],xmm8[1],xmm2[2],xmm8[2],xmm2[3],xmm8[3]8070; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm3[0],xmm7[0],xmm3[1],xmm7[1],xmm3[2],xmm7[2],xmm3[3],xmm7[3]8071; AVX512-FCP-NEXT:    vpermt2d %zmm4, %zmm16, %zmm58072; AVX512-FCP-NEXT:    vmovdqa 64(%rcx), %xmm48073; AVX512-FCP-NEXT:    vmovdqa 64(%rdx), %xmm68074; AVX512-FCP-NEXT:    vmovdqa 64(%rsi), %xmm138075; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm08076; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]8077; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]8078; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm18, %zmm98079; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, %zmm9 {%k1}8080; AVX512-FCP-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8081; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]8082; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm7[4],xmm3[5],xmm7[5],xmm3[6],xmm7[6],xmm3[7],xmm7[7]8083; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm16, %zmm28084; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]8085; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm0[4],xmm13[4],xmm0[5],xmm13[5],xmm0[6],xmm13[6],xmm0[7],xmm13[7]8086; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm18, %zmm158087; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, %zmm15 {%k1}8088; AVX512-FCP-NEXT:    vmovdqa 32(%r10), %ymm88089; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %ymm48090; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm4[0],ymm8[0],ymm4[1],ymm8[1],ymm4[2],ymm8[2],ymm4[3],ymm8[3],ymm4[8],ymm8[8],ymm4[9],ymm8[9],ymm4[10],ymm8[10],ymm4[11],ymm8[11]8091; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %ymm38092; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %ymm58093; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[8],ymm3[8],ymm5[9],ymm3[9],ymm5[10],ymm3[10],ymm5[11],ymm3[11]8094; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm68095; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %ymm28096; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %ymm78097; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm7[0],ymm2[0],ymm7[1],ymm2[1],ymm7[2],ymm2[2],ymm7[3],ymm2[3],ymm7[8],ymm2[8],ymm7[9],ymm2[9],ymm7[10],ymm2[10],ymm7[11],ymm2[11]8098; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %ymm18099; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm08100; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]8101; AVX512-FCP-NEXT:    vpermt2d %zmm9, %zmm19, %zmm148102; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, %zmm14 {%k1}8103; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm4[4],ymm8[4],ymm4[5],ymm8[5],ymm4[6],ymm8[6],ymm4[7],ymm8[7],ymm4[12],ymm8[12],ymm4[13],ymm8[13],ymm4[14],ymm8[14],ymm4[15],ymm8[15]8104; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm3[4],ymm5[5],ymm3[5],ymm5[6],ymm3[6],ymm5[7],ymm3[7],ymm5[12],ymm3[12],ymm5[13],ymm3[13],ymm5[14],ymm3[14],ymm5[15],ymm3[15]8105; AVX512-FCP-NEXT:    vpermt2d %zmm4, %zmm17, %zmm58106; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm7[4],ymm2[4],ymm7[5],ymm2[5],ymm7[6],ymm2[6],ymm7[7],ymm2[7],ymm7[12],ymm2[12],ymm7[13],ymm2[13],ymm7[14],ymm2[14],ymm7[15],ymm2[15]8107; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm13 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]8108; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm19, %zmm138109; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, %zmm13 {%k1}8110; AVX512-FCP-NEXT:    vmovdqa 32(%r10), %xmm108111; AVX512-FCP-NEXT:    vmovdqa 32(%rax), %xmm38112; AVX512-FCP-NEXT:    vmovdqa 32(%r9), %xmm48113; AVX512-FCP-NEXT:    vmovdqa 32(%r8), %xmm58114; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]8115; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]8116; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm16, %zmm68117; AVX512-FCP-NEXT:    vmovdqa 32(%rcx), %xmm78118; AVX512-FCP-NEXT:    vmovdqa 32(%rdx), %xmm88119; AVX512-FCP-NEXT:    vmovdqa 32(%rsi), %xmm98120; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %xmm08121; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]8122; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3]8123; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm18, %zmm128124; AVX512-FCP-NEXT:    vmovdqa64 %zmm6, %zmm12 {%k1}8125; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]8126; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]8127; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm16, %zmm38128; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]8129; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]8130; AVX512-FCP-NEXT:    vpermt2d %zmm1, %zmm18, %zmm48131; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}8132; AVX512-FCP-NEXT:    vmovdqa (%r10), %ymm108133; AVX512-FCP-NEXT:    vmovdqa (%rax), %ymm38134; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm3[0],ymm10[0],ymm3[1],ymm10[1],ymm3[2],ymm10[2],ymm3[3],ymm10[3],ymm3[8],ymm10[8],ymm3[9],ymm10[9],ymm3[10],ymm10[10],ymm3[11],ymm10[11]8135; AVX512-FCP-NEXT:    vmovdqa (%r9), %ymm58136; AVX512-FCP-NEXT:    vmovdqa (%r8), %ymm68137; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm6[0],ymm5[0],ymm6[1],ymm5[1],ymm6[2],ymm5[2],ymm6[3],ymm5[3],ymm6[8],ymm5[8],ymm6[9],ymm5[9],ymm6[10],ymm5[10],ymm6[11],ymm5[11]8138; AVX512-FCP-NEXT:    vpermt2d %zmm0, %zmm17, %zmm78139; AVX512-FCP-NEXT:    vmovdqa (%rcx), %ymm88140; AVX512-FCP-NEXT:    vmovdqa (%rdx), %ymm98141; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]8142; AVX512-FCP-NEXT:    vmovdqa (%rsi), %ymm28143; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm18144; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]8145; AVX512-FCP-NEXT:    vpermt2d %zmm11, %zmm19, %zmm08146; AVX512-FCP-NEXT:    vmovdqa64 %zmm7, %zmm0 {%k1}8147; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm10[4],ymm3[5],ymm10[5],ymm3[6],ymm10[6],ymm3[7],ymm10[7],ymm3[12],ymm10[12],ymm3[13],ymm10[13],ymm3[14],ymm10[14],ymm3[15],ymm10[15]8148; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]8149; AVX512-FCP-NEXT:    vpermt2d %zmm3, %zmm17, %zmm58150; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15]8151; AVX512-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]8152; AVX512-FCP-NEXT:    vpermt2d %zmm3, %zmm19, %zmm18153; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, %zmm1 {%k1}8154; AVX512-FCP-NEXT:    vmovdqa64 %xmm21, %xmm28155; AVX512-FCP-NEXT:    vmovdqa64 %xmm22, %xmm38156; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]8157; AVX512-FCP-NEXT:    vmovdqa64 %xmm23, %xmm38158; AVX512-FCP-NEXT:    vmovdqa64 %xmm24, %xmm58159; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]8160; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm16, %zmm38161; AVX512-FCP-NEXT:    vmovdqa64 %xmm25, %xmm28162; AVX512-FCP-NEXT:    vmovdqa64 %xmm26, %xmm58163; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]8164; AVX512-FCP-NEXT:    vmovdqa64 %xmm27, %xmm58165; AVX512-FCP-NEXT:    vmovdqa64 %xmm28, %xmm68166; AVX512-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]8167; AVX512-FCP-NEXT:    vpermt2d %zmm2, %zmm18, %zmm58168; AVX512-FCP-NEXT:    vmovdqa64 %zmm3, %zmm5 {%k1}8169; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax8170; AVX512-FCP-NEXT:    vmovdqa64 %zmm5, (%rax)8171; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, 192(%rax)8172; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 128(%rax)8173; AVX512-FCP-NEXT:    vmovdqa64 %zmm4, 320(%rax)8174; AVX512-FCP-NEXT:    vmovdqa64 %zmm12, 256(%rax)8175; AVX512-FCP-NEXT:    vmovdqa64 %zmm13, 448(%rax)8176; AVX512-FCP-NEXT:    vmovdqa64 %zmm14, 384(%rax)8177; AVX512-FCP-NEXT:    vmovdqa64 %zmm15, 576(%rax)8178; AVX512-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8179; AVX512-FCP-NEXT:    vmovaps %zmm0, 512(%rax)8180; AVX512-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8181; AVX512-FCP-NEXT:    vmovaps %zmm0, 704(%rax)8182; AVX512-FCP-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload8183; AVX512-FCP-NEXT:    vmovaps %zmm0, 640(%rax)8184; AVX512-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8185; AVX512-FCP-NEXT:    vmovaps %zmm0, 832(%rax)8186; AVX512-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8187; AVX512-FCP-NEXT:    vmovaps %zmm0, 768(%rax)8188; AVX512-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8189; AVX512-FCP-NEXT:    vmovaps %zmm0, 960(%rax)8190; AVX512-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8191; AVX512-FCP-NEXT:    vmovaps %zmm0, 896(%rax)8192; AVX512-FCP-NEXT:    vmovdqa64 %zmm20, 64(%rax)8193; AVX512-FCP-NEXT:    addq $328, %rsp # imm = 0x1488194; AVX512-FCP-NEXT:    vzeroupper8195; AVX512-FCP-NEXT:    retq8196;8197; AVX512DQ-LABEL: store_i16_stride8_vf64:8198; AVX512DQ:       # %bb.0:8199; AVX512DQ-NEXT:    subq $392, %rsp # imm = 0x1888200; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax8201; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %r108202; AVX512DQ-NEXT:    vmovdqa (%r10), %xmm18203; AVX512DQ-NEXT:    vmovdqa (%rax), %xmm28204; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]8205; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm208206; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm218207; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm28208; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm38209; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]8210; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm228211; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm238212; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]8213; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm16, %zmm18214; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm28215; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm38216; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]8217; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm248218; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm258219; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm38220; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm48221; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]8222; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm268223; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm278224; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]8225; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm17, %zmm28226; AVX512DQ-NEXT:    movb $-86, %r11b8227; AVX512DQ-NEXT:    kmovw %r11d, %k18228; AVX512DQ-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}8229; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8230; AVX512DQ-NEXT:    vmovdqa 96(%r10), %ymm08231; AVX512DQ-NEXT:    vmovdqa 96(%rax), %ymm18232; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]8233; AVX512DQ-NEXT:    vmovdqa 96(%r9), %ymm38234; AVX512DQ-NEXT:    vmovdqa 96(%r8), %ymm48235; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]8236; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]8237; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm18, %zmm58238; AVX512DQ-NEXT:    vmovdqa 96(%rcx), %ymm28239; AVX512DQ-NEXT:    vmovdqa 96(%rdx), %ymm68240; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]8241; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm6[0],ymm2[0],ymm6[1],ymm2[1],ymm6[2],ymm2[2],ymm6[3],ymm2[3],ymm6[8],ymm2[8],ymm6[9],ymm2[9],ymm6[10],ymm2[10],ymm6[11],ymm2[11]8242; AVX512DQ-NEXT:    vmovdqa 96(%rsi), %ymm128243; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %ymm138244; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]8245; AVX512DQ-NEXT:    vpermt2d %zmm10, %zmm19, %zmm78246; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}8247; AVX512DQ-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8248; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]8249; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]8250; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm18, %zmm18251; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm6[4],ymm2[4],ymm6[5],ymm2[5],ymm6[6],ymm2[6],ymm6[7],ymm2[7],ymm6[12],ymm2[12],ymm6[13],ymm2[13],ymm6[14],ymm2[14],ymm6[15],ymm2[15]8252; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm12[4],ymm13[5],ymm12[5],ymm13[6],ymm12[6],ymm13[7],ymm12[7],ymm13[12],ymm12[12],ymm13[13],ymm12[13],ymm13[14],ymm12[14],ymm13[15],ymm12[15]8253; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm19, %zmm28254; AVX512DQ-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}8255; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8256; AVX512DQ-NEXT:    vmovdqa 96(%r10), %xmm08257; AVX512DQ-NEXT:    vmovdqa 96(%rax), %xmm18258; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]8259; AVX512DQ-NEXT:    vmovdqa 96(%r9), %xmm38260; AVX512DQ-NEXT:    vmovdqa 96(%r8), %xmm48261; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]8262; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm16, %zmm58263; AVX512DQ-NEXT:    vmovdqa 96(%rcx), %xmm28264; AVX512DQ-NEXT:    vmovdqa 96(%rdx), %xmm68265; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]8266; AVX512DQ-NEXT:    vmovdqa 96(%rsi), %xmm138267; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm148268; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]8269; AVX512DQ-NEXT:    vpermt2d %zmm10, %zmm17, %zmm78270; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}8271; AVX512DQ-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8272; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]8273; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]8274; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm16, %zmm18275; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]8276; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]8277; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm17, %zmm28278; AVX512DQ-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}8279; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8280; AVX512DQ-NEXT:    vmovdqa 64(%r10), %ymm68281; AVX512DQ-NEXT:    vmovdqa 64(%rax), %ymm58282; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11]8283; AVX512DQ-NEXT:    vmovdqa 64(%r9), %ymm38284; AVX512DQ-NEXT:    vmovdqa 64(%r8), %ymm48285; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm10 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]8286; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm18, %zmm108287; AVX512DQ-NEXT:    vmovdqa 64(%rcx), %ymm28288; AVX512DQ-NEXT:    vmovdqa 64(%rdx), %ymm138289; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm13[0],ymm2[0],ymm13[1],ymm2[1],ymm13[2],ymm2[2],ymm13[3],ymm2[3],ymm13[8],ymm2[8],ymm13[9],ymm2[9],ymm13[10],ymm2[10],ymm13[11],ymm2[11]8290; AVX512DQ-NEXT:    vmovdqa 64(%rsi), %ymm18291; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %ymm08292; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]8293; AVX512DQ-NEXT:    vpermt2d %zmm14, %zmm19, %zmm78294; AVX512DQ-NEXT:    vmovdqa64 %zmm10, %zmm7 {%k1}8295; AVX512DQ-NEXT:    vmovdqu64 %zmm7, (%rsp) # 64-byte Spill8296; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15]8297; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]8298; AVX512DQ-NEXT:    vpermt2d %zmm5, %zmm18, %zmm38299; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm2[4],ymm13[5],ymm2[5],ymm13[6],ymm2[6],ymm13[7],ymm2[7],ymm13[12],ymm2[12],ymm13[13],ymm2[13],ymm13[14],ymm2[14],ymm13[15],ymm2[15]8300; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]8301; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm19, %zmm08302; AVX512DQ-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k1}8303; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8304; AVX512DQ-NEXT:    vmovdqa 64(%r10), %xmm78305; AVX512DQ-NEXT:    vmovdqa 64(%rax), %xmm28306; AVX512DQ-NEXT:    vmovdqa 64(%r9), %xmm88307; AVX512DQ-NEXT:    vmovdqa 64(%r8), %xmm38308; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm7[0],xmm2[1],xmm7[1],xmm2[2],xmm7[2],xmm2[3],xmm7[3]8309; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3]8310; AVX512DQ-NEXT:    vpermt2d %zmm4, %zmm16, %zmm58311; AVX512DQ-NEXT:    vmovdqa 64(%rcx), %xmm48312; AVX512DQ-NEXT:    vmovdqa 64(%rdx), %xmm68313; AVX512DQ-NEXT:    vmovdqa 64(%rsi), %xmm148314; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm08315; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]8316; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]8317; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm17, %zmm98318; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm9 {%k1}8319; AVX512DQ-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8320; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm7[4],xmm2[5],xmm7[5],xmm2[6],xmm7[6],xmm2[7],xmm7[7]8321; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm8[4],xmm3[5],xmm8[5],xmm3[6],xmm8[6],xmm3[7],xmm8[7]8322; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm16, %zmm28323; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]8324; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]8325; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm17, %zmm158326; AVX512DQ-NEXT:    vmovdqa64 %zmm2, %zmm15 {%k1}8327; AVX512DQ-NEXT:    vmovdqa 32(%r10), %ymm88328; AVX512DQ-NEXT:    vmovdqa 32(%rax), %ymm48329; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm4[0],ymm8[0],ymm4[1],ymm8[1],ymm4[2],ymm8[2],ymm4[3],ymm8[3],ymm4[8],ymm8[8],ymm4[9],ymm8[9],ymm4[10],ymm8[10],ymm4[11],ymm8[11]8330; AVX512DQ-NEXT:    vmovdqa 32(%r9), %ymm38331; AVX512DQ-NEXT:    vmovdqa 32(%r8), %ymm58332; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[8],ymm3[8],ymm5[9],ymm3[9],ymm5[10],ymm3[10],ymm5[11],ymm3[11]8333; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm18, %zmm68334; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %ymm28335; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %ymm78336; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm7[0],ymm2[0],ymm7[1],ymm2[1],ymm7[2],ymm2[2],ymm7[3],ymm2[3],ymm7[8],ymm2[8],ymm7[9],ymm2[9],ymm7[10],ymm2[10],ymm7[11],ymm2[11]8337; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %ymm18338; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm08339; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]8340; AVX512DQ-NEXT:    vpermt2d %zmm9, %zmm19, %zmm148341; AVX512DQ-NEXT:    vmovdqa64 %zmm6, %zmm14 {%k1}8342; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm4[4],ymm8[4],ymm4[5],ymm8[5],ymm4[6],ymm8[6],ymm4[7],ymm8[7],ymm4[12],ymm8[12],ymm4[13],ymm8[13],ymm4[14],ymm8[14],ymm4[15],ymm8[15]8343; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm3[4],ymm5[5],ymm3[5],ymm5[6],ymm3[6],ymm5[7],ymm3[7],ymm5[12],ymm3[12],ymm5[13],ymm3[13],ymm5[14],ymm3[14],ymm5[15],ymm3[15]8344; AVX512DQ-NEXT:    vpermt2d %zmm4, %zmm18, %zmm58345; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm7[4],ymm2[4],ymm7[5],ymm2[5],ymm7[6],ymm2[6],ymm7[7],ymm2[7],ymm7[12],ymm2[12],ymm7[13],ymm2[13],ymm7[14],ymm2[14],ymm7[15],ymm2[15]8346; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm13 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]8347; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm19, %zmm138348; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm13 {%k1}8349; AVX512DQ-NEXT:    vmovdqa 32(%r10), %xmm108350; AVX512DQ-NEXT:    vmovdqa 32(%rax), %xmm38351; AVX512DQ-NEXT:    vmovdqa 32(%r9), %xmm48352; AVX512DQ-NEXT:    vmovdqa 32(%r8), %xmm58353; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]8354; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]8355; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm16, %zmm68356; AVX512DQ-NEXT:    vmovdqa 32(%rcx), %xmm78357; AVX512DQ-NEXT:    vmovdqa 32(%rdx), %xmm88358; AVX512DQ-NEXT:    vmovdqa 32(%rsi), %xmm98359; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %xmm08360; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]8361; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3]8362; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm17, %zmm128363; AVX512DQ-NEXT:    vmovdqa64 %zmm6, %zmm12 {%k1}8364; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]8365; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]8366; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm16, %zmm38367; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]8368; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]8369; AVX512DQ-NEXT:    vpermt2d %zmm1, %zmm17, %zmm48370; AVX512DQ-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}8371; AVX512DQ-NEXT:    vmovdqa (%r10), %ymm108372; AVX512DQ-NEXT:    vmovdqa (%rax), %ymm38373; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm3[0],ymm10[0],ymm3[1],ymm10[1],ymm3[2],ymm10[2],ymm3[3],ymm10[3],ymm3[8],ymm10[8],ymm3[9],ymm10[9],ymm3[10],ymm10[10],ymm3[11],ymm10[11]8374; AVX512DQ-NEXT:    vmovdqa (%r9), %ymm58375; AVX512DQ-NEXT:    vmovdqa (%r8), %ymm68376; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm6[0],ymm5[0],ymm6[1],ymm5[1],ymm6[2],ymm5[2],ymm6[3],ymm5[3],ymm6[8],ymm5[8],ymm6[9],ymm5[9],ymm6[10],ymm5[10],ymm6[11],ymm5[11]8377; AVX512DQ-NEXT:    vpermt2d %zmm0, %zmm18, %zmm78378; AVX512DQ-NEXT:    vmovdqa (%rcx), %ymm88379; AVX512DQ-NEXT:    vmovdqa (%rdx), %ymm98380; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]8381; AVX512DQ-NEXT:    vmovdqa (%rsi), %ymm28382; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm18383; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]8384; AVX512DQ-NEXT:    vpermt2d %zmm11, %zmm19, %zmm08385; AVX512DQ-NEXT:    vmovdqa64 %zmm7, %zmm0 {%k1}8386; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm10[4],ymm3[5],ymm10[5],ymm3[6],ymm10[6],ymm3[7],ymm10[7],ymm3[12],ymm10[12],ymm3[13],ymm10[13],ymm3[14],ymm10[14],ymm3[15],ymm10[15]8387; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]8388; AVX512DQ-NEXT:    vpermt2d %zmm3, %zmm18, %zmm58389; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15]8390; AVX512DQ-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]8391; AVX512DQ-NEXT:    vpermt2d %zmm3, %zmm19, %zmm18392; AVX512DQ-NEXT:    vmovdqa64 %zmm5, %zmm1 {%k1}8393; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm28394; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm38395; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]8396; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm38397; AVX512DQ-NEXT:    vmovdqa64 %xmm23, %xmm58398; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]8399; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm16, %zmm38400; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm28401; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm58402; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]8403; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm58404; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm68405; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]8406; AVX512DQ-NEXT:    vpermt2d %zmm2, %zmm17, %zmm58407; AVX512DQ-NEXT:    vmovdqa64 %zmm3, %zmm5 {%k1}8408; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax8409; AVX512DQ-NEXT:    vmovdqa64 %zmm5, (%rax)8410; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 192(%rax)8411; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 128(%rax)8412; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 320(%rax)8413; AVX512DQ-NEXT:    vmovdqa64 %zmm12, 256(%rax)8414; AVX512DQ-NEXT:    vmovdqa64 %zmm13, 448(%rax)8415; AVX512DQ-NEXT:    vmovdqa64 %zmm14, 384(%rax)8416; AVX512DQ-NEXT:    vmovdqa64 %zmm15, 576(%rax)8417; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8418; AVX512DQ-NEXT:    vmovaps %zmm0, 512(%rax)8419; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8420; AVX512DQ-NEXT:    vmovaps %zmm0, 704(%rax)8421; AVX512DQ-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload8422; AVX512DQ-NEXT:    vmovaps %zmm0, 640(%rax)8423; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8424; AVX512DQ-NEXT:    vmovaps %zmm0, 832(%rax)8425; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8426; AVX512DQ-NEXT:    vmovaps %zmm0, 768(%rax)8427; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8428; AVX512DQ-NEXT:    vmovaps %zmm0, 960(%rax)8429; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8430; AVX512DQ-NEXT:    vmovaps %zmm0, 896(%rax)8431; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8432; AVX512DQ-NEXT:    vmovaps %zmm0, 64(%rax)8433; AVX512DQ-NEXT:    addq $392, %rsp # imm = 0x1888434; AVX512DQ-NEXT:    vzeroupper8435; AVX512DQ-NEXT:    retq8436;8437; AVX512DQ-FCP-LABEL: store_i16_stride8_vf64:8438; AVX512DQ-FCP:       # %bb.0:8439; AVX512DQ-FCP-NEXT:    subq $328, %rsp # imm = 0x1488440; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax8441; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r108442; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %xmm18443; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %xmm28444; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]8445; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm218446; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm228447; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm28448; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm38449; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]8450; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm238451; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm248452; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]8453; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm16, %zmm18454; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm28455; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm38456; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]8457; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm258458; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm2, %xmm268459; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm38460; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm48461; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]8462; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm4, %xmm278463; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm288464; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm20 = [0,16,1,0,1,17,0,0,2,18,3,0,3,19,0,0]8465; AVX512DQ-FCP-NEXT:    vpermi2d %zmm0, %zmm2, %zmm208466; AVX512DQ-FCP-NEXT:    movb $-86, %r11b8467; AVX512DQ-FCP-NEXT:    kmovw %r11d, %k18468; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, %zmm20 {%k1}8469; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r10), %ymm08470; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rax), %ymm18471; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]8472; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r9), %ymm38473; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r8), %ymm48474; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]8475; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm17 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]8476; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm58477; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rcx), %ymm28478; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdx), %ymm68479; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm6[0],ymm2[0],ymm6[1],ymm2[1],ymm6[2],ymm2[2],ymm6[3],ymm2[3],ymm6[8],ymm2[8],ymm6[9],ymm2[9],ymm6[10],ymm2[10],ymm6[11],ymm2[11]8480; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]8481; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %ymm128482; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %ymm138483; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]8484; AVX512DQ-FCP-NEXT:    vpermt2d %zmm11, %zmm19, %zmm78485; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}8486; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8487; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]8488; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]8489; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm17, %zmm18490; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm6[4],ymm2[4],ymm6[5],ymm2[5],ymm6[6],ymm2[6],ymm6[7],ymm2[7],ymm6[12],ymm2[12],ymm6[13],ymm2[13],ymm6[14],ymm2[14],ymm6[15],ymm2[15]8491; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm12[4],ymm13[5],ymm12[5],ymm13[6],ymm12[6],ymm13[7],ymm12[7],ymm13[12],ymm12[12],ymm13[13],ymm12[13],ymm13[14],ymm12[14],ymm13[15],ymm12[15]8492; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm19, %zmm28493; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}8494; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8495; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r10), %xmm08496; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rax), %xmm18497; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]8498; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r9), %xmm38499; AVX512DQ-FCP-NEXT:    vmovdqa 96(%r8), %xmm48500; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]8501; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm16, %zmm58502; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rcx), %xmm28503; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} zmm18 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]8504; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdx), %xmm68505; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]8506; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rsi), %xmm138507; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm158508; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3]8509; AVX512DQ-FCP-NEXT:    vpermt2d %zmm12, %zmm18, %zmm78510; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}8511; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8512; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]8513; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]8514; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm16, %zmm18515; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]8516; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm15[4],xmm13[4],xmm15[5],xmm13[5],xmm15[6],xmm13[6],xmm15[7],xmm13[7]8517; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm18, %zmm28518; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}8519; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8520; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r10), %ymm68521; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rax), %ymm58522; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11]8523; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r9), %ymm38524; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r8), %ymm48525; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm12 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]8526; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm128527; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rcx), %ymm28528; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdx), %ymm138529; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm15 = ymm13[0],ymm2[0],ymm13[1],ymm2[1],ymm13[2],ymm2[2],ymm13[3],ymm2[3],ymm13[8],ymm2[8],ymm13[9],ymm2[9],ymm13[10],ymm2[10],ymm13[11],ymm2[11]8530; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rsi), %ymm18531; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %ymm08532; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]8533; AVX512DQ-FCP-NEXT:    vpermt2d %zmm15, %zmm19, %zmm78534; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm12, %zmm7 {%k1}8535; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm7, (%rsp) # 64-byte Spill8536; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15]8537; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]8538; AVX512DQ-FCP-NEXT:    vpermt2d %zmm5, %zmm17, %zmm38539; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm2[4],ymm13[5],ymm2[5],ymm13[6],ymm2[6],ymm13[7],ymm2[7],ymm13[12],ymm2[12],ymm13[13],ymm2[13],ymm13[14],ymm2[14],ymm13[15],ymm2[15]8540; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]8541; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm19, %zmm08542; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r10), %xmm88543; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k1}8544; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8545; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rax), %xmm28546; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r9), %xmm78547; AVX512DQ-FCP-NEXT:    vmovdqa 64(%r8), %xmm38548; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm8[0],xmm2[1],xmm8[1],xmm2[2],xmm8[2],xmm2[3],xmm8[3]8549; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm3[0],xmm7[0],xmm3[1],xmm7[1],xmm3[2],xmm7[2],xmm3[3],xmm7[3]8550; AVX512DQ-FCP-NEXT:    vpermt2d %zmm4, %zmm16, %zmm58551; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rcx), %xmm48552; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdx), %xmm68553; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rsi), %xmm138554; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %xmm08555; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]8556; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]8557; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm18, %zmm98558; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, %zmm9 {%k1}8559; AVX512DQ-FCP-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8560; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]8561; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm7[4],xmm3[5],xmm7[5],xmm3[6],xmm7[6],xmm3[7],xmm7[7]8562; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm16, %zmm28563; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]8564; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm0[4],xmm13[4],xmm0[5],xmm13[5],xmm0[6],xmm13[6],xmm0[7],xmm13[7]8565; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm18, %zmm158566; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, %zmm15 {%k1}8567; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r10), %ymm88568; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %ymm48569; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm4[0],ymm8[0],ymm4[1],ymm8[1],ymm4[2],ymm8[2],ymm4[3],ymm8[3],ymm4[8],ymm8[8],ymm4[9],ymm8[9],ymm4[10],ymm8[10],ymm4[11],ymm8[11]8570; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %ymm38571; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %ymm58572; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[8],ymm3[8],ymm5[9],ymm3[9],ymm5[10],ymm3[10],ymm5[11],ymm3[11]8573; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm17, %zmm68574; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %ymm28575; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %ymm78576; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm9 = ymm7[0],ymm2[0],ymm7[1],ymm2[1],ymm7[2],ymm2[2],ymm7[3],ymm2[3],ymm7[8],ymm2[8],ymm7[9],ymm2[9],ymm7[10],ymm2[10],ymm7[11],ymm2[11]8577; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %ymm18578; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm08579; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm14 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]8580; AVX512DQ-FCP-NEXT:    vpermt2d %zmm9, %zmm19, %zmm148581; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, %zmm14 {%k1}8582; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm4[4],ymm8[4],ymm4[5],ymm8[5],ymm4[6],ymm8[6],ymm4[7],ymm8[7],ymm4[12],ymm8[12],ymm4[13],ymm8[13],ymm4[14],ymm8[14],ymm4[15],ymm8[15]8583; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm3[4],ymm5[5],ymm3[5],ymm5[6],ymm3[6],ymm5[7],ymm3[7],ymm5[12],ymm3[12],ymm5[13],ymm3[13],ymm5[14],ymm3[14],ymm5[15],ymm3[15]8584; AVX512DQ-FCP-NEXT:    vpermt2d %zmm4, %zmm17, %zmm58585; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm7[4],ymm2[4],ymm7[5],ymm2[5],ymm7[6],ymm2[6],ymm7[7],ymm2[7],ymm7[12],ymm2[12],ymm7[13],ymm2[13],ymm7[14],ymm2[14],ymm7[15],ymm2[15]8586; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm13 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]8587; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm19, %zmm138588; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, %zmm13 {%k1}8589; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r10), %xmm108590; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rax), %xmm38591; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r9), %xmm48592; AVX512DQ-FCP-NEXT:    vmovdqa 32(%r8), %xmm58593; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]8594; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]8595; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm16, %zmm68596; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rcx), %xmm78597; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdx), %xmm88598; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rsi), %xmm98599; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %xmm08600; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]8601; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3]8602; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm18, %zmm128603; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm6, %zmm12 {%k1}8604; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]8605; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]8606; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm16, %zmm38607; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]8608; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]8609; AVX512DQ-FCP-NEXT:    vpermt2d %zmm1, %zmm18, %zmm48610; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}8611; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %ymm108612; AVX512DQ-FCP-NEXT:    vmovdqa (%rax), %ymm38613; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm3[0],ymm10[0],ymm3[1],ymm10[1],ymm3[2],ymm10[2],ymm3[3],ymm10[3],ymm3[8],ymm10[8],ymm3[9],ymm10[9],ymm3[10],ymm10[10],ymm3[11],ymm10[11]8614; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %ymm58615; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %ymm68616; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm6[0],ymm5[0],ymm6[1],ymm5[1],ymm6[2],ymm5[2],ymm6[3],ymm5[3],ymm6[8],ymm5[8],ymm6[9],ymm5[9],ymm6[10],ymm5[10],ymm6[11],ymm5[11]8617; AVX512DQ-FCP-NEXT:    vpermt2d %zmm0, %zmm17, %zmm78618; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %ymm88619; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %ymm98620; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm11 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]8621; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %ymm28622; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm18623; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]8624; AVX512DQ-FCP-NEXT:    vpermt2d %zmm11, %zmm19, %zmm08625; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, %zmm0 {%k1}8626; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm10[4],ymm3[5],ymm10[5],ymm3[6],ymm10[6],ymm3[7],ymm10[7],ymm3[12],ymm10[12],ymm3[13],ymm10[13],ymm3[14],ymm10[14],ymm3[15],ymm10[15]8627; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]8628; AVX512DQ-FCP-NEXT:    vpermt2d %zmm3, %zmm17, %zmm58629; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15]8630; AVX512DQ-FCP-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]8631; AVX512DQ-FCP-NEXT:    vpermt2d %zmm3, %zmm19, %zmm18632; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, %zmm1 {%k1}8633; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm21, %xmm28634; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm22, %xmm38635; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]8636; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm23, %xmm38637; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm24, %xmm58638; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]8639; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm16, %zmm38640; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm25, %xmm28641; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm26, %xmm58642; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]8643; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm27, %xmm58644; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm28, %xmm68645; AVX512DQ-FCP-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]8646; AVX512DQ-FCP-NEXT:    vpermt2d %zmm2, %zmm18, %zmm58647; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm3, %zmm5 {%k1}8648; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax8649; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm5, (%rax)8650; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, 192(%rax)8651; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 128(%rax)8652; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm4, 320(%rax)8653; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm12, 256(%rax)8654; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm13, 448(%rax)8655; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm14, 384(%rax)8656; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm15, 576(%rax)8657; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8658; AVX512DQ-FCP-NEXT:    vmovaps %zmm0, 512(%rax)8659; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8660; AVX512DQ-FCP-NEXT:    vmovaps %zmm0, 704(%rax)8661; AVX512DQ-FCP-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload8662; AVX512DQ-FCP-NEXT:    vmovaps %zmm0, 640(%rax)8663; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8664; AVX512DQ-FCP-NEXT:    vmovaps %zmm0, 832(%rax)8665; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8666; AVX512DQ-FCP-NEXT:    vmovaps %zmm0, 768(%rax)8667; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8668; AVX512DQ-FCP-NEXT:    vmovaps %zmm0, 960(%rax)8669; AVX512DQ-FCP-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8670; AVX512DQ-FCP-NEXT:    vmovaps %zmm0, 896(%rax)8671; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm20, 64(%rax)8672; AVX512DQ-FCP-NEXT:    addq $328, %rsp # imm = 0x1488673; AVX512DQ-FCP-NEXT:    vzeroupper8674; AVX512DQ-FCP-NEXT:    retq8675;8676; AVX512BW-LABEL: store_i16_stride8_vf64:8677; AVX512BW:       # %bb.0:8678; AVX512BW-NEXT:    subq $2056, %rsp # imm = 0x8088679; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax8680; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r108681; AVX512BW-NEXT:    vmovdqa64 (%r8), %zmm18682; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm278683; AVX512BW-NEXT:    vmovdqa64 (%r10), %zmm28684; AVX512BW-NEXT:    vmovdqa64 64(%r10), %zmm08685; AVX512BW-NEXT:    vmovdqa64 (%rax), %zmm308686; AVX512BW-NEXT:    vmovdqa64 64(%rax), %zmm288687; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [0,0,0,0,0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39]8688; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm58689; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm48690; AVX512BW-NEXT:    vpermt2w %zmm30, %zmm3, %zmm58691; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8692; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,0,0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0]8693; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm58694; AVX512BW-NEXT:    vpermt2w %zmm27, %zmm2, %zmm58695; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8696; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [0,0,0,0,0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35]8697; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm68698; AVX512BW-NEXT:    vpermt2w %zmm30, %zmm5, %zmm68699; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8700; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [0,0,0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0]8701; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm78702; AVX512BW-NEXT:    vpermt2w %zmm27, %zmm6, %zmm78703; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8704; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [0,0,0,0,0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47]8705; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm88706; AVX512BW-NEXT:    vpermt2w %zmm30, %zmm7, %zmm88707; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8708; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm8 = [0,0,0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0]8709; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm98710; AVX512BW-NEXT:    vpermt2w %zmm27, %zmm8, %zmm98711; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8712; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,0,0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43]8713; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm108714; AVX512BW-NEXT:    vpermt2w %zmm30, %zmm9, %zmm108715; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8716; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,0,0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0]8717; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm118718; AVX512BW-NEXT:    vpermt2w %zmm27, %zmm10, %zmm118719; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8720; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [0,0,0,0,0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55]8721; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm128722; AVX512BW-NEXT:    vpermt2w %zmm30, %zmm11, %zmm128723; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8724; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0]8725; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm138726; AVX512BW-NEXT:    vpermt2w %zmm27, %zmm12, %zmm138727; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8728; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51]8729; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm148730; AVX512BW-NEXT:    vpermt2w %zmm30, %zmm13, %zmm148731; AVX512BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8732; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [0,0,0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0]8733; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm15 = [0,0,0,0,0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63]8734; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm168735; AVX512BW-NEXT:    vpermt2w %zmm30, %zmm15, %zmm168736; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8737; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm31 = [0,0,0,0,0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59]8738; AVX512BW-NEXT:    vpermt2w %zmm30, %zmm31, %zmm48739; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8740; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm48741; AVX512BW-NEXT:    vpermt2w %zmm27, %zmm14, %zmm48742; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8743; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm38744; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8745; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm58746; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8747; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm78748; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8749; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm98750; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8751; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm118752; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8753; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm138754; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8755; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm158756; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8757; AVX512BW-NEXT:    vpermt2w %zmm28, %zmm31, %zmm08758; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8759; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm28 = [0,0,0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0]8760; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm318761; AVX512BW-NEXT:    vpermt2w %zmm27, %zmm28, %zmm318762; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm0 = [0,0,0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0]8763; AVX512BW-NEXT:    vpermt2w %zmm27, %zmm0, %zmm18764; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8765; AVX512BW-NEXT:    vmovdqa64 64(%r8), %zmm278766; AVX512BW-NEXT:    vmovdqa64 64(%r9), %zmm18767; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm27, %zmm28768; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8769; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm27, %zmm68770; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8771; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm27, %zmm88772; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8773; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm27, %zmm108774; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8775; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm27, %zmm128776; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8777; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm27, %zmm148778; AVX512BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8779; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm27, %zmm288780; AVX512BW-NEXT:    vpermt2w %zmm1, %zmm0, %zmm278781; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm168782; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm08783; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0,0,0]8784; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm208785; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm2, %zmm208786; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0,0,0]8787; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm218788; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm3, %zmm218789; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0,0,0]8790; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm228791; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm4, %zmm228792; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0,0,0]8793; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm238794; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm6, %zmm238795; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0,0,0]8796; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm248797; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm7, %zmm248798; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm29 = [0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0,0,0]8799; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm258800; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm29, %zmm258801; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm30 = [0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0,0,0]8802; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm268803; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm30, %zmm268804; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0,0,0]8805; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm1, %zmm168806; AVX512BW-NEXT:    vmovdqa64 64(%rdx), %zmm58807; AVX512BW-NEXT:    vmovdqa64 64(%rcx), %zmm08808; AVX512BW-NEXT:    vpermi2w %zmm0, %zmm5, %zmm28809; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8810; AVX512BW-NEXT:    vpermi2w %zmm0, %zmm5, %zmm38811; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8812; AVX512BW-NEXT:    vpermi2w %zmm0, %zmm5, %zmm48813; AVX512BW-NEXT:    vmovdqu64 %zmm4, (%rsp) # 64-byte Spill8814; AVX512BW-NEXT:    vpermi2w %zmm0, %zmm5, %zmm68815; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8816; AVX512BW-NEXT:    vpermi2w %zmm0, %zmm5, %zmm78817; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8818; AVX512BW-NEXT:    vpermi2w %zmm0, %zmm5, %zmm298819; AVX512BW-NEXT:    vpermi2w %zmm0, %zmm5, %zmm308820; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm1, %zmm58821; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm38822; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm08823; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0,0,0,0,0]8824; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm188825; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm12, %zmm188826; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0,0,0,0,0]8827; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm178828; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm10, %zmm178829; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm8 = [12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0,0,0,0,0]8830; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm158831; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm8, %zmm158832; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0,0,0,0,0]8833; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm148834; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm7, %zmm148835; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0,0,0,0,0]8836; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm138837; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm6, %zmm138838; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0,0,0,0,0]8839; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm118840; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm4, %zmm118841; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0,0,0,0,0]8842; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm98843; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm2, %zmm98844; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm19 = [24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0,0,0,0,0]8845; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm19, %zmm38846; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm18847; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %zmm08848; AVX512BW-NEXT:    vpermi2w %zmm0, %zmm1, %zmm128849; AVX512BW-NEXT:    vpermi2w %zmm0, %zmm1, %zmm108850; AVX512BW-NEXT:    vpermi2w %zmm0, %zmm1, %zmm88851; AVX512BW-NEXT:    vpermi2w %zmm0, %zmm1, %zmm78852; AVX512BW-NEXT:    vpermi2w %zmm0, %zmm1, %zmm68853; AVX512BW-NEXT:    vpermi2w %zmm0, %zmm1, %zmm48854; AVX512BW-NEXT:    vpermi2w %zmm0, %zmm1, %zmm28855; AVX512BW-NEXT:    vpermt2w %zmm0, %zmm19, %zmm18856; AVX512BW-NEXT:    movw $-30584, %ax # imm = 0x88888857; AVX512BW-NEXT:    kmovd %eax, %k18858; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8859; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload8860; AVX512BW-NEXT:    vmovdqa32 %zmm19, %zmm0 {%k1}8861; AVX512BW-NEXT:    movw $8738, %ax # imm = 0x22228862; AVX512BW-NEXT:    kmovd %eax, %k28863; AVX512BW-NEXT:    vmovdqa32 %zmm20, %zmm18 {%k2}8864; AVX512BW-NEXT:    movb $-86, %al8865; AVX512BW-NEXT:    kmovd %eax, %k38866; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k3}8867; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8868; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload8869; AVX512BW-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}8870; AVX512BW-NEXT:    vmovdqa32 %zmm21, %zmm17 {%k2}8871; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm17 {%k3}8872; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8873; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload8874; AVX512BW-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}8875; AVX512BW-NEXT:    vmovdqa32 %zmm22, %zmm15 {%k2}8876; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm15 {%k3}8877; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8878; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload8879; AVX512BW-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}8880; AVX512BW-NEXT:    vmovdqa32 %zmm23, %zmm14 {%k2}8881; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm14 {%k3}8882; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8883; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload8884; AVX512BW-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}8885; AVX512BW-NEXT:    vmovdqa32 %zmm24, %zmm13 {%k2}8886; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm13 {%k3}8887; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8888; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload8889; AVX512BW-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}8890; AVX512BW-NEXT:    vmovdqa32 %zmm25, %zmm11 {%k2}8891; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm11 {%k3}8892; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8893; AVX512BW-NEXT:    vmovdqa32 %zmm0, %zmm31 {%k1}8894; AVX512BW-NEXT:    vmovdqa32 %zmm26, %zmm9 {%k2}8895; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm9 {%k3}8896; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8897; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload8898; AVX512BW-NEXT:    vmovdqa32 %zmm19, %zmm0 {%k1}8899; AVX512BW-NEXT:    vmovdqa32 %zmm16, %zmm3 {%k2}8900; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k3}8901; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8902; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload8903; AVX512BW-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}8904; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload8905; AVX512BW-NEXT:    vmovdqa32 %zmm16, %zmm12 {%k2}8906; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm12 {%k3}8907; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8908; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload8909; AVX512BW-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}8910; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload8911; AVX512BW-NEXT:    vmovdqa32 %zmm16, %zmm10 {%k2}8912; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k3}8913; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8914; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload8915; AVX512BW-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}8916; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm16 # 64-byte Reload8917; AVX512BW-NEXT:    vmovdqa32 %zmm16, %zmm8 {%k2}8918; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k3}8919; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8920; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload8921; AVX512BW-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}8922; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload8923; AVX512BW-NEXT:    vmovdqa32 %zmm16, %zmm7 {%k2}8924; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k3}8925; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8926; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload8927; AVX512BW-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}8928; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload8929; AVX512BW-NEXT:    vmovdqa32 %zmm16, %zmm6 {%k2}8930; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k3}8931; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8932; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload8933; AVX512BW-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}8934; AVX512BW-NEXT:    vmovdqa32 %zmm29, %zmm4 {%k2}8935; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k3}8936; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8937; AVX512BW-NEXT:    vmovdqa32 %zmm0, %zmm28 {%k1}8938; AVX512BW-NEXT:    vmovdqa32 %zmm30, %zmm2 {%k2}8939; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm2 {%k3}8940; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload8941; AVX512BW-NEXT:    vmovdqa32 %zmm0, %zmm27 {%k1}8942; AVX512BW-NEXT:    vmovdqa32 %zmm5, %zmm1 {%k2}8943; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm1 {%k3}8944; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax8945; AVX512BW-NEXT:    vmovdqa64 %zmm1, 896(%rax)8946; AVX512BW-NEXT:    vmovdqa64 %zmm2, 960(%rax)8947; AVX512BW-NEXT:    vmovdqa64 %zmm4, 768(%rax)8948; AVX512BW-NEXT:    vmovdqa64 %zmm6, 832(%rax)8949; AVX512BW-NEXT:    vmovdqa64 %zmm7, 640(%rax)8950; AVX512BW-NEXT:    vmovdqa64 %zmm8, 704(%rax)8951; AVX512BW-NEXT:    vmovdqa64 %zmm10, 512(%rax)8952; AVX512BW-NEXT:    vmovdqa64 %zmm12, 576(%rax)8953; AVX512BW-NEXT:    vmovdqa64 %zmm3, 384(%rax)8954; AVX512BW-NEXT:    vmovdqa64 %zmm9, 448(%rax)8955; AVX512BW-NEXT:    vmovdqa64 %zmm11, 256(%rax)8956; AVX512BW-NEXT:    vmovdqa64 %zmm13, 320(%rax)8957; AVX512BW-NEXT:    vmovdqa64 %zmm14, 128(%rax)8958; AVX512BW-NEXT:    vmovdqa64 %zmm15, 192(%rax)8959; AVX512BW-NEXT:    vmovdqa64 %zmm17, (%rax)8960; AVX512BW-NEXT:    vmovdqa64 %zmm18, 64(%rax)8961; AVX512BW-NEXT:    addq $2056, %rsp # imm = 0x8088962; AVX512BW-NEXT:    vzeroupper8963; AVX512BW-NEXT:    retq8964;8965; AVX512BW-FCP-LABEL: store_i16_stride8_vf64:8966; AVX512BW-FCP:       # %bb.0:8967; AVX512BW-FCP-NEXT:    subq $2056, %rsp # imm = 0x8088968; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax8969; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r108970; AVX512BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm18971; AVX512BW-FCP-NEXT:    vmovdqa64 (%r9), %zmm278972; AVX512BW-FCP-NEXT:    vmovdqa64 (%r10), %zmm28973; AVX512BW-FCP-NEXT:    vmovdqa64 64(%r10), %zmm08974; AVX512BW-FCP-NEXT:    vmovdqa64 (%rax), %zmm308975; AVX512BW-FCP-NEXT:    vmovdqa64 64(%rax), %zmm288976; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [0,0,0,0,0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39]8977; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm2, %zmm58978; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm2, %zmm48979; AVX512BW-FCP-NEXT:    vpermt2w %zmm30, %zmm3, %zmm58980; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8981; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,0,0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0]8982; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm1, %zmm58983; AVX512BW-FCP-NEXT:    vpermt2w %zmm27, %zmm2, %zmm58984; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8985; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [0,0,0,0,0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35]8986; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm68987; AVX512BW-FCP-NEXT:    vpermt2w %zmm30, %zmm5, %zmm68988; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8989; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [0,0,0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0]8990; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm1, %zmm78991; AVX512BW-FCP-NEXT:    vpermt2w %zmm27, %zmm6, %zmm78992; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8993; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [0,0,0,0,0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47]8994; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm88995; AVX512BW-FCP-NEXT:    vpermt2w %zmm30, %zmm7, %zmm88996; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill8997; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm8 = [0,0,0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0]8998; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm1, %zmm98999; AVX512BW-FCP-NEXT:    vpermt2w %zmm27, %zmm8, %zmm99000; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9001; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,0,0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43]9002; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm109003; AVX512BW-FCP-NEXT:    vpermt2w %zmm30, %zmm9, %zmm109004; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9005; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,0,0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0]9006; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm1, %zmm119007; AVX512BW-FCP-NEXT:    vpermt2w %zmm27, %zmm10, %zmm119008; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9009; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [0,0,0,0,0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55]9010; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm129011; AVX512BW-FCP-NEXT:    vpermt2w %zmm30, %zmm11, %zmm129012; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9013; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0]9014; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm1, %zmm139015; AVX512BW-FCP-NEXT:    vpermt2w %zmm27, %zmm12, %zmm139016; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9017; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51]9018; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm149019; AVX512BW-FCP-NEXT:    vpermt2w %zmm30, %zmm13, %zmm149020; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9021; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [0,0,0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0]9022; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm15 = [0,0,0,0,0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63]9023; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm169024; AVX512BW-FCP-NEXT:    vpermt2w %zmm30, %zmm15, %zmm169025; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9026; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm31 = [0,0,0,0,0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59]9027; AVX512BW-FCP-NEXT:    vpermt2w %zmm30, %zmm31, %zmm49028; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9029; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm1, %zmm49030; AVX512BW-FCP-NEXT:    vpermt2w %zmm27, %zmm14, %zmm49031; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9032; AVX512BW-FCP-NEXT:    vpermi2w %zmm28, %zmm0, %zmm39033; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9034; AVX512BW-FCP-NEXT:    vpermi2w %zmm28, %zmm0, %zmm59035; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9036; AVX512BW-FCP-NEXT:    vpermi2w %zmm28, %zmm0, %zmm79037; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9038; AVX512BW-FCP-NEXT:    vpermi2w %zmm28, %zmm0, %zmm99039; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9040; AVX512BW-FCP-NEXT:    vpermi2w %zmm28, %zmm0, %zmm119041; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9042; AVX512BW-FCP-NEXT:    vpermi2w %zmm28, %zmm0, %zmm139043; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9044; AVX512BW-FCP-NEXT:    vpermi2w %zmm28, %zmm0, %zmm159045; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9046; AVX512BW-FCP-NEXT:    vpermt2w %zmm28, %zmm31, %zmm09047; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9048; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm28 = [0,0,0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0]9049; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm1, %zmm319050; AVX512BW-FCP-NEXT:    vpermt2w %zmm27, %zmm28, %zmm319051; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm0 = [0,0,0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0]9052; AVX512BW-FCP-NEXT:    vpermt2w %zmm27, %zmm0, %zmm19053; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9054; AVX512BW-FCP-NEXT:    vmovdqa64 64(%r8), %zmm279055; AVX512BW-FCP-NEXT:    vmovdqa64 64(%r9), %zmm19056; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm27, %zmm29057; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9058; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm27, %zmm69059; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9060; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm27, %zmm89061; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9062; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm27, %zmm109063; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9064; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm27, %zmm129065; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9066; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm27, %zmm149067; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9068; AVX512BW-FCP-NEXT:    vpermi2w %zmm1, %zmm27, %zmm289069; AVX512BW-FCP-NEXT:    vpermt2w %zmm1, %zmm0, %zmm279070; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdx), %zmm169071; AVX512BW-FCP-NEXT:    vmovdqa64 (%rcx), %zmm09072; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0,0,0]9073; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm16, %zmm209074; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm2, %zmm209075; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0,0,0]9076; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm16, %zmm219077; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm3, %zmm219078; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0,0,0]9079; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm16, %zmm229080; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm4, %zmm229081; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0,0,0]9082; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm16, %zmm239083; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm6, %zmm239084; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0,0,0]9085; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm16, %zmm249086; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm7, %zmm249087; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm29 = [0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0,0,0]9088; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm16, %zmm259089; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm29, %zmm259090; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm30 = [0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0,0,0]9091; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm16, %zmm269092; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm30, %zmm269093; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0,0,0]9094; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm1, %zmm169095; AVX512BW-FCP-NEXT:    vmovdqa64 64(%rdx), %zmm59096; AVX512BW-FCP-NEXT:    vmovdqa64 64(%rcx), %zmm09097; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm5, %zmm29098; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9099; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm5, %zmm39100; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9101; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm5, %zmm49102; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm4, (%rsp) # 64-byte Spill9103; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm5, %zmm69104; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9105; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm5, %zmm79106; AVX512BW-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9107; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm5, %zmm299108; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm5, %zmm309109; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm1, %zmm59110; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdi), %zmm39111; AVX512BW-FCP-NEXT:    vmovdqa64 (%rsi), %zmm09112; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0,0,0,0,0]9113; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, %zmm189114; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm12, %zmm189115; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0,0,0,0,0]9116; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, %zmm179117; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm10, %zmm179118; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm8 = [12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0,0,0,0,0]9119; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, %zmm159120; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm8, %zmm159121; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0,0,0,0,0]9122; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, %zmm149123; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm7, %zmm149124; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0,0,0,0,0]9125; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, %zmm139126; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm6, %zmm139127; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0,0,0,0,0]9128; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, %zmm119129; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm4, %zmm119130; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0,0,0,0,0]9131; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, %zmm99132; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm2, %zmm99133; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm19 = [24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0,0,0,0,0]9134; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm19, %zmm39135; AVX512BW-FCP-NEXT:    vmovdqa64 64(%rdi), %zmm19136; AVX512BW-FCP-NEXT:    vmovdqa64 64(%rsi), %zmm09137; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm1, %zmm129138; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm1, %zmm109139; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm1, %zmm89140; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm1, %zmm79141; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm1, %zmm69142; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm1, %zmm49143; AVX512BW-FCP-NEXT:    vpermi2w %zmm0, %zmm1, %zmm29144; AVX512BW-FCP-NEXT:    vpermt2w %zmm0, %zmm19, %zmm19145; AVX512BW-FCP-NEXT:    movw $-30584, %ax # imm = 0x88889146; AVX512BW-FCP-NEXT:    kmovd %eax, %k19147; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9148; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9149; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm19, %zmm0 {%k1}9150; AVX512BW-FCP-NEXT:    movw $8738, %ax # imm = 0x22229151; AVX512BW-FCP-NEXT:    kmovd %eax, %k29152; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm20, %zmm18 {%k2}9153; AVX512BW-FCP-NEXT:    movb $-86, %al9154; AVX512BW-FCP-NEXT:    kmovd %eax, %k39155; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k3}9156; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9157; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9158; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}9159; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm21, %zmm17 {%k2}9160; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm19, %zmm17 {%k3}9161; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9162; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9163; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}9164; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm22, %zmm15 {%k2}9165; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm19, %zmm15 {%k3}9166; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9167; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9168; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}9169; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm23, %zmm14 {%k2}9170; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm19, %zmm14 {%k3}9171; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9172; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9173; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}9174; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm24, %zmm13 {%k2}9175; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm19, %zmm13 {%k3}9176; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9177; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9178; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}9179; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm25, %zmm11 {%k2}9180; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm19, %zmm11 {%k3}9181; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9182; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm31 {%k1}9183; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm26, %zmm9 {%k2}9184; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm31, %zmm9 {%k3}9185; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9186; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9187; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm19, %zmm0 {%k1}9188; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm3 {%k2}9189; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k3}9190; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9191; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9192; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9193; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9194; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm12 {%k2}9195; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm12 {%k3}9196; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9197; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9198; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9199; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9200; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm10 {%k2}9201; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k3}9202; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9203; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9204; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9205; AVX512BW-FCP-NEXT:    vmovdqu64 (%rsp), %zmm16 # 64-byte Reload9206; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm8 {%k2}9207; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k3}9208; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9209; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9210; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9211; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9212; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm7 {%k2}9213; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k3}9214; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9215; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9216; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9217; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9218; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm6 {%k2}9219; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k3}9220; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9221; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9222; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9223; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm29, %zmm4 {%k2}9224; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k3}9225; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9226; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm28 {%k1}9227; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm30, %zmm2 {%k2}9228; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm28, %zmm2 {%k3}9229; AVX512BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9230; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm27 {%k1}9231; AVX512BW-FCP-NEXT:    vmovdqa32 %zmm5, %zmm1 {%k2}9232; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm27, %zmm1 {%k3}9233; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax9234; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm1, 896(%rax)9235; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm2, 960(%rax)9236; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, 768(%rax)9237; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm6, 832(%rax)9238; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm7, 640(%rax)9239; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm8, 704(%rax)9240; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm10, 512(%rax)9241; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm12, 576(%rax)9242; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, 384(%rax)9243; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm9, 448(%rax)9244; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm11, 256(%rax)9245; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm13, 320(%rax)9246; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm14, 128(%rax)9247; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm15, 192(%rax)9248; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm17, (%rax)9249; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm18, 64(%rax)9250; AVX512BW-FCP-NEXT:    addq $2056, %rsp # imm = 0x8089251; AVX512BW-FCP-NEXT:    vzeroupper9252; AVX512BW-FCP-NEXT:    retq9253;9254; AVX512DQ-BW-LABEL: store_i16_stride8_vf64:9255; AVX512DQ-BW:       # %bb.0:9256; AVX512DQ-BW-NEXT:    subq $2056, %rsp # imm = 0x8089257; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax9258; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %r109259; AVX512DQ-BW-NEXT:    vmovdqa64 (%r8), %zmm19260; AVX512DQ-BW-NEXT:    vmovdqa64 (%r9), %zmm279261; AVX512DQ-BW-NEXT:    vmovdqa64 (%r10), %zmm29262; AVX512DQ-BW-NEXT:    vmovdqa64 64(%r10), %zmm09263; AVX512DQ-BW-NEXT:    vmovdqa64 (%rax), %zmm309264; AVX512DQ-BW-NEXT:    vmovdqa64 64(%rax), %zmm289265; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [0,0,0,0,0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39]9266; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm2, %zmm59267; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm2, %zmm49268; AVX512DQ-BW-NEXT:    vpermt2w %zmm30, %zmm3, %zmm59269; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9270; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,0,0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0]9271; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm1, %zmm59272; AVX512DQ-BW-NEXT:    vpermt2w %zmm27, %zmm2, %zmm59273; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9274; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [0,0,0,0,0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35]9275; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, %zmm69276; AVX512DQ-BW-NEXT:    vpermt2w %zmm30, %zmm5, %zmm69277; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9278; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [0,0,0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0]9279; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm1, %zmm79280; AVX512DQ-BW-NEXT:    vpermt2w %zmm27, %zmm6, %zmm79281; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9282; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [0,0,0,0,0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47]9283; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, %zmm89284; AVX512DQ-BW-NEXT:    vpermt2w %zmm30, %zmm7, %zmm89285; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9286; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm8 = [0,0,0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0]9287; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm1, %zmm99288; AVX512DQ-BW-NEXT:    vpermt2w %zmm27, %zmm8, %zmm99289; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9290; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,0,0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43]9291; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, %zmm109292; AVX512DQ-BW-NEXT:    vpermt2w %zmm30, %zmm9, %zmm109293; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9294; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,0,0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0]9295; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm1, %zmm119296; AVX512DQ-BW-NEXT:    vpermt2w %zmm27, %zmm10, %zmm119297; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9298; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [0,0,0,0,0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55]9299; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, %zmm129300; AVX512DQ-BW-NEXT:    vpermt2w %zmm30, %zmm11, %zmm129301; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9302; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0]9303; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm1, %zmm139304; AVX512DQ-BW-NEXT:    vpermt2w %zmm27, %zmm12, %zmm139305; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9306; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51]9307; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, %zmm149308; AVX512DQ-BW-NEXT:    vpermt2w %zmm30, %zmm13, %zmm149309; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9310; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [0,0,0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0]9311; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm15 = [0,0,0,0,0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63]9312; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, %zmm169313; AVX512DQ-BW-NEXT:    vpermt2w %zmm30, %zmm15, %zmm169314; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9315; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm31 = [0,0,0,0,0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59]9316; AVX512DQ-BW-NEXT:    vpermt2w %zmm30, %zmm31, %zmm49317; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9318; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm1, %zmm49319; AVX512DQ-BW-NEXT:    vpermt2w %zmm27, %zmm14, %zmm49320; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9321; AVX512DQ-BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm39322; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9323; AVX512DQ-BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm59324; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9325; AVX512DQ-BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm79326; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9327; AVX512DQ-BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm99328; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9329; AVX512DQ-BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm119330; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9331; AVX512DQ-BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm139332; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9333; AVX512DQ-BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm159334; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9335; AVX512DQ-BW-NEXT:    vpermt2w %zmm28, %zmm31, %zmm09336; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9337; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm28 = [0,0,0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0]9338; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm1, %zmm319339; AVX512DQ-BW-NEXT:    vpermt2w %zmm27, %zmm28, %zmm319340; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm0 = [0,0,0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0]9341; AVX512DQ-BW-NEXT:    vpermt2w %zmm27, %zmm0, %zmm19342; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9343; AVX512DQ-BW-NEXT:    vmovdqa64 64(%r8), %zmm279344; AVX512DQ-BW-NEXT:    vmovdqa64 64(%r9), %zmm19345; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm27, %zmm29346; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9347; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm27, %zmm69348; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9349; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm27, %zmm89350; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9351; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm27, %zmm109352; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9353; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm27, %zmm129354; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9355; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm27, %zmm149356; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9357; AVX512DQ-BW-NEXT:    vpermi2w %zmm1, %zmm27, %zmm289358; AVX512DQ-BW-NEXT:    vpermt2w %zmm1, %zmm0, %zmm279359; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdx), %zmm169360; AVX512DQ-BW-NEXT:    vmovdqa64 (%rcx), %zmm09361; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0,0,0]9362; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm16, %zmm209363; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm2, %zmm209364; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0,0,0]9365; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm16, %zmm219366; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm3, %zmm219367; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0,0,0]9368; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm16, %zmm229369; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm4, %zmm229370; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0,0,0]9371; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm16, %zmm239372; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm6, %zmm239373; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0,0,0]9374; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm16, %zmm249375; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm7, %zmm249376; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm29 = [0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0,0,0]9377; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm16, %zmm259378; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm29, %zmm259379; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm30 = [0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0,0,0]9380; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm16, %zmm269381; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm30, %zmm269382; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0,0,0]9383; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm1, %zmm169384; AVX512DQ-BW-NEXT:    vmovdqa64 64(%rdx), %zmm59385; AVX512DQ-BW-NEXT:    vmovdqa64 64(%rcx), %zmm09386; AVX512DQ-BW-NEXT:    vpermi2w %zmm0, %zmm5, %zmm29387; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9388; AVX512DQ-BW-NEXT:    vpermi2w %zmm0, %zmm5, %zmm39389; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9390; AVX512DQ-BW-NEXT:    vpermi2w %zmm0, %zmm5, %zmm49391; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm4, (%rsp) # 64-byte Spill9392; AVX512DQ-BW-NEXT:    vpermi2w %zmm0, %zmm5, %zmm69393; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9394; AVX512DQ-BW-NEXT:    vpermi2w %zmm0, %zmm5, %zmm79395; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9396; AVX512DQ-BW-NEXT:    vpermi2w %zmm0, %zmm5, %zmm299397; AVX512DQ-BW-NEXT:    vpermi2w %zmm0, %zmm5, %zmm309398; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm1, %zmm59399; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdi), %zmm39400; AVX512DQ-BW-NEXT:    vmovdqa64 (%rsi), %zmm09401; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0,0,0,0,0]9402; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm3, %zmm189403; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm12, %zmm189404; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0,0,0,0,0]9405; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm3, %zmm179406; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm10, %zmm179407; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm8 = [12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0,0,0,0,0]9408; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm3, %zmm159409; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm8, %zmm159410; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0,0,0,0,0]9411; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm3, %zmm149412; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm7, %zmm149413; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0,0,0,0,0]9414; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm3, %zmm139415; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm6, %zmm139416; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0,0,0,0,0]9417; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm3, %zmm119418; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm4, %zmm119419; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0,0,0,0,0]9420; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm3, %zmm99421; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm2, %zmm99422; AVX512DQ-BW-NEXT:    vpmovsxbw {{.*#+}} zmm19 = [24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0,0,0,0,0]9423; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm19, %zmm39424; AVX512DQ-BW-NEXT:    vmovdqa64 64(%rdi), %zmm19425; AVX512DQ-BW-NEXT:    vmovdqa64 64(%rsi), %zmm09426; AVX512DQ-BW-NEXT:    vpermi2w %zmm0, %zmm1, %zmm129427; AVX512DQ-BW-NEXT:    vpermi2w %zmm0, %zmm1, %zmm109428; AVX512DQ-BW-NEXT:    vpermi2w %zmm0, %zmm1, %zmm89429; AVX512DQ-BW-NEXT:    vpermi2w %zmm0, %zmm1, %zmm79430; AVX512DQ-BW-NEXT:    vpermi2w %zmm0, %zmm1, %zmm69431; AVX512DQ-BW-NEXT:    vpermi2w %zmm0, %zmm1, %zmm49432; AVX512DQ-BW-NEXT:    vpermi2w %zmm0, %zmm1, %zmm29433; AVX512DQ-BW-NEXT:    vpermt2w %zmm0, %zmm19, %zmm19434; AVX512DQ-BW-NEXT:    movw $-30584, %ax # imm = 0x88889435; AVX512DQ-BW-NEXT:    kmovd %eax, %k19436; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9437; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9438; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm19, %zmm0 {%k1}9439; AVX512DQ-BW-NEXT:    movw $8738, %ax # imm = 0x22229440; AVX512DQ-BW-NEXT:    kmovd %eax, %k29441; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm20, %zmm18 {%k2}9442; AVX512DQ-BW-NEXT:    movb $-86, %al9443; AVX512DQ-BW-NEXT:    kmovd %eax, %k39444; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k3}9445; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9446; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9447; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}9448; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm21, %zmm17 {%k2}9449; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm19, %zmm17 {%k3}9450; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9451; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9452; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}9453; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm22, %zmm15 {%k2}9454; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm19, %zmm15 {%k3}9455; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9456; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9457; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}9458; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm23, %zmm14 {%k2}9459; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm19, %zmm14 {%k3}9460; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9461; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9462; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}9463; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm24, %zmm13 {%k2}9464; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm19, %zmm13 {%k3}9465; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9466; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9467; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}9468; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm25, %zmm11 {%k2}9469; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm19, %zmm11 {%k3}9470; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9471; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm0, %zmm31 {%k1}9472; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm26, %zmm9 {%k2}9473; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm31, %zmm9 {%k3}9474; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9475; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9476; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm19, %zmm0 {%k1}9477; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm16, %zmm3 {%k2}9478; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k3}9479; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9480; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9481; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9482; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9483; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm16, %zmm12 {%k2}9484; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, %zmm12 {%k3}9485; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9486; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9487; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9488; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9489; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm16, %zmm10 {%k2}9490; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k3}9491; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9492; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9493; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9494; AVX512DQ-BW-NEXT:    vmovdqu64 (%rsp), %zmm16 # 64-byte Reload9495; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm16, %zmm8 {%k2}9496; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k3}9497; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9498; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9499; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9500; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9501; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm16, %zmm7 {%k2}9502; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k3}9503; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9504; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9505; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9506; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9507; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm16, %zmm6 {%k2}9508; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k3}9509; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9510; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9511; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9512; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm29, %zmm4 {%k2}9513; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k3}9514; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9515; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm0, %zmm28 {%k1}9516; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm30, %zmm2 {%k2}9517; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm28, %zmm2 {%k3}9518; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9519; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm0, %zmm27 {%k1}9520; AVX512DQ-BW-NEXT:    vmovdqa32 %zmm5, %zmm1 {%k2}9521; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm27, %zmm1 {%k3}9522; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax9523; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm1, 896(%rax)9524; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm2, 960(%rax)9525; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm4, 768(%rax)9526; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm6, 832(%rax)9527; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm7, 640(%rax)9528; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm8, 704(%rax)9529; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm10, 512(%rax)9530; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm12, 576(%rax)9531; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm3, 384(%rax)9532; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm9, 448(%rax)9533; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm11, 256(%rax)9534; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm13, 320(%rax)9535; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm14, 128(%rax)9536; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm15, 192(%rax)9537; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm17, (%rax)9538; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm18, 64(%rax)9539; AVX512DQ-BW-NEXT:    addq $2056, %rsp # imm = 0x8089540; AVX512DQ-BW-NEXT:    vzeroupper9541; AVX512DQ-BW-NEXT:    retq9542;9543; AVX512DQ-BW-FCP-LABEL: store_i16_stride8_vf64:9544; AVX512DQ-BW-FCP:       # %bb.0:9545; AVX512DQ-BW-FCP-NEXT:    subq $2056, %rsp # imm = 0x8089546; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax9547; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r109548; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm19549; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r9), %zmm279550; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r10), %zmm29551; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 64(%r10), %zmm09552; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rax), %zmm309553; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 64(%rax), %zmm289554; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [0,0,0,0,0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39]9555; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm2, %zmm59556; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm2, %zmm49557; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm30, %zmm3, %zmm59558; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9559; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,0,0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0]9560; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm1, %zmm59561; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm27, %zmm2, %zmm59562; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9563; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [0,0,0,0,0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35]9564; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm69565; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm30, %zmm5, %zmm69566; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9567; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [0,0,0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0]9568; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm1, %zmm79569; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm27, %zmm6, %zmm79570; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9571; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [0,0,0,0,0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47]9572; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm89573; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm30, %zmm7, %zmm89574; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9575; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm8 = [0,0,0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0]9576; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm1, %zmm99577; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm27, %zmm8, %zmm99578; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9579; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm9 = [0,0,0,0,0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43]9580; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm109581; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm30, %zmm9, %zmm109582; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9583; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,0,0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0]9584; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm1, %zmm119585; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm27, %zmm10, %zmm119586; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9587; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm11 = [0,0,0,0,0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55]9588; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm129589; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm30, %zmm11, %zmm129590; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9591; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [0,0,0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0]9592; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm1, %zmm139593; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm27, %zmm12, %zmm139594; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9595; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm13 = [0,0,0,0,0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51]9596; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm149597; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm30, %zmm13, %zmm149598; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9599; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm14 = [0,0,0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0]9600; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm15 = [0,0,0,0,0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63]9601; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, %zmm169602; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm30, %zmm15, %zmm169603; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9604; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm31 = [0,0,0,0,0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59]9605; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm30, %zmm31, %zmm49606; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9607; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm1, %zmm49608; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm27, %zmm14, %zmm49609; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9610; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm28, %zmm0, %zmm39611; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9612; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm28, %zmm0, %zmm59613; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9614; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm28, %zmm0, %zmm79615; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9616; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm28, %zmm0, %zmm99617; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9618; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm28, %zmm0, %zmm119619; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9620; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm28, %zmm0, %zmm139621; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9622; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm28, %zmm0, %zmm159623; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9624; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm28, %zmm31, %zmm09625; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9626; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm28 = [0,0,0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0]9627; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm1, %zmm319628; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm27, %zmm28, %zmm319629; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm0 = [0,0,0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0]9630; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm27, %zmm0, %zmm19631; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9632; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 64(%r8), %zmm279633; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 64(%r9), %zmm19634; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm27, %zmm29635; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9636; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm27, %zmm69637; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9638; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm27, %zmm89639; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9640; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm27, %zmm109641; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9642; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm27, %zmm129643; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9644; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm27, %zmm149645; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9646; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm1, %zmm27, %zmm289647; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm1, %zmm0, %zmm279648; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdx), %zmm169649; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rcx), %zmm09650; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [0,0,4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0,0,0]9651; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm16, %zmm209652; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm2, %zmm209653; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm3 = [0,0,0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0,0,0]9654; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm16, %zmm219655; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm3, %zmm219656; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [0,0,12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0,0,0]9657; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm16, %zmm229658; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm4, %zmm229659; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [0,0,8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0,0,0]9660; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm16, %zmm239661; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm6, %zmm239662; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [0,0,20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0,0,0]9663; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm16, %zmm249664; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm7, %zmm249665; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm29 = [0,0,16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0,0,0]9666; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm16, %zmm259667; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm29, %zmm259668; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm30 = [0,0,28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0,0,0]9669; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm16, %zmm269670; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm30, %zmm269671; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [0,0,24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0,0,0]9672; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm1, %zmm169673; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 64(%rdx), %zmm59674; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 64(%rcx), %zmm09675; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm0, %zmm5, %zmm29676; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9677; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm0, %zmm5, %zmm39678; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9679; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm0, %zmm5, %zmm49680; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm4, (%rsp) # 64-byte Spill9681; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm0, %zmm5, %zmm69682; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9683; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm0, %zmm5, %zmm79684; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill9685; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm0, %zmm5, %zmm299686; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm0, %zmm5, %zmm309687; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm1, %zmm59688; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdi), %zmm39689; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rsi), %zmm09690; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm12 = [4,36,0,0,0,0,0,0,5,37,0,0,0,0,0,0,6,38,0,0,0,0,0,0,7,39,0,0,0,0,0,0]9691; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm3, %zmm189692; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm12, %zmm189693; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm10 = [0,32,0,0,0,0,0,0,1,33,0,0,0,0,0,0,2,34,0,0,0,0,0,0,3,35,0,0,0,0,0,0]9694; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm3, %zmm179695; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm10, %zmm179696; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm8 = [12,44,0,0,0,0,0,0,13,45,0,0,0,0,0,0,14,46,0,0,0,0,0,0,15,47,0,0,0,0,0,0]9697; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm3, %zmm159698; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm8, %zmm159699; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm7 = [8,40,0,0,0,0,0,0,9,41,0,0,0,0,0,0,10,42,0,0,0,0,0,0,11,43,0,0,0,0,0,0]9700; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm3, %zmm149701; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm7, %zmm149702; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm6 = [20,52,0,0,0,0,0,0,21,53,0,0,0,0,0,0,22,54,0,0,0,0,0,0,23,55,0,0,0,0,0,0]9703; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm3, %zmm139704; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm6, %zmm139705; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [16,48,0,0,0,0,0,0,17,49,0,0,0,0,0,0,18,50,0,0,0,0,0,0,19,51,0,0,0,0,0,0]9706; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm3, %zmm119707; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm4, %zmm119708; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [28,60,0,0,0,0,0,0,29,61,0,0,0,0,0,0,30,62,0,0,0,0,0,0,31,63,0,0,0,0,0,0]9709; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm3, %zmm99710; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm2, %zmm99711; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm19 = [24,56,0,0,0,0,0,0,25,57,0,0,0,0,0,0,26,58,0,0,0,0,0,0,27,59,0,0,0,0,0,0]9712; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm19, %zmm39713; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 64(%rdi), %zmm19714; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 64(%rsi), %zmm09715; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm0, %zmm1, %zmm129716; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm0, %zmm1, %zmm109717; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm0, %zmm1, %zmm89718; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm0, %zmm1, %zmm79719; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm0, %zmm1, %zmm69720; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm0, %zmm1, %zmm49721; AVX512DQ-BW-FCP-NEXT:    vpermi2w %zmm0, %zmm1, %zmm29722; AVX512DQ-BW-FCP-NEXT:    vpermt2w %zmm0, %zmm19, %zmm19723; AVX512DQ-BW-FCP-NEXT:    movw $-30584, %ax # imm = 0x88889724; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k19725; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9726; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9727; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm19, %zmm0 {%k1}9728; AVX512DQ-BW-FCP-NEXT:    movw $8738, %ax # imm = 0x22229729; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k29730; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm20, %zmm18 {%k2}9731; AVX512DQ-BW-FCP-NEXT:    movb $-86, %al9732; AVX512DQ-BW-FCP-NEXT:    kmovd %eax, %k39733; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k3}9734; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9735; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9736; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}9737; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm21, %zmm17 {%k2}9738; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm19, %zmm17 {%k3}9739; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9740; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9741; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}9742; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm22, %zmm15 {%k2}9743; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm19, %zmm15 {%k3}9744; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9745; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9746; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}9747; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm23, %zmm14 {%k2}9748; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm19, %zmm14 {%k3}9749; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9750; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9751; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}9752; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm24, %zmm13 {%k2}9753; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm19, %zmm13 {%k3}9754; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9755; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9756; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm19 {%k1}9757; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm25, %zmm11 {%k2}9758; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm19, %zmm11 {%k3}9759; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9760; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm31 {%k1}9761; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm26, %zmm9 {%k2}9762; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm31, %zmm9 {%k3}9763; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9764; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload9765; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm19, %zmm0 {%k1}9766; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm3 {%k2}9767; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k3}9768; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9769; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9770; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9771; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9772; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm12 {%k2}9773; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm12 {%k3}9774; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9775; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9776; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9777; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9778; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm10 {%k2}9779; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k3}9780; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9781; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9782; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9783; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 (%rsp), %zmm16 # 64-byte Reload9784; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm8 {%k2}9785; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k3}9786; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9787; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9788; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9789; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9790; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm7 {%k2}9791; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k3}9792; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9793; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9794; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9795; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9796; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm6 {%k2}9797; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k3}9798; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9799; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload9800; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm16, %zmm0 {%k1}9801; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm29, %zmm4 {%k2}9802; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k3}9803; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9804; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm28 {%k1}9805; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm30, %zmm2 {%k2}9806; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm28, %zmm2 {%k3}9807; AVX512DQ-BW-FCP-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload9808; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm0, %zmm27 {%k1}9809; AVX512DQ-BW-FCP-NEXT:    vmovdqa32 %zmm5, %zmm1 {%k2}9810; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm27, %zmm1 {%k3}9811; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax9812; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm1, 896(%rax)9813; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm2, 960(%rax)9814; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, 768(%rax)9815; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm6, 832(%rax)9816; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm7, 640(%rax)9817; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm8, 704(%rax)9818; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm10, 512(%rax)9819; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm12, 576(%rax)9820; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm3, 384(%rax)9821; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm9, 448(%rax)9822; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm11, 256(%rax)9823; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm13, 320(%rax)9824; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm14, 128(%rax)9825; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm15, 192(%rax)9826; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm17, (%rax)9827; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm18, 64(%rax)9828; AVX512DQ-BW-FCP-NEXT:    addq $2056, %rsp # imm = 0x8089829; AVX512DQ-BW-FCP-NEXT:    vzeroupper9830; AVX512DQ-BW-FCP-NEXT:    retq9831  %in.vec0 = load <64 x i16>, ptr %in.vecptr0, align 649832  %in.vec1 = load <64 x i16>, ptr %in.vecptr1, align 649833  %in.vec2 = load <64 x i16>, ptr %in.vecptr2, align 649834  %in.vec3 = load <64 x i16>, ptr %in.vecptr3, align 649835  %in.vec4 = load <64 x i16>, ptr %in.vecptr4, align 649836  %in.vec5 = load <64 x i16>, ptr %in.vecptr5, align 649837  %in.vec6 = load <64 x i16>, ptr %in.vecptr6, align 649838  %in.vec7 = load <64 x i16>, ptr %in.vecptr7, align 649839  %1 = shufflevector <64 x i16> %in.vec0, <64 x i16> %in.vec1, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>9840  %2 = shufflevector <64 x i16> %in.vec2, <64 x i16> %in.vec3, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>9841  %3 = shufflevector <64 x i16> %in.vec4, <64 x i16> %in.vec5, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>9842  %4 = shufflevector <64 x i16> %in.vec6, <64 x i16> %in.vec7, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>9843  %5 = shufflevector <128 x i16> %1, <128 x i16> %2, <256 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127, i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143, i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151, i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159, i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191, i32 192, i32 193, i32 194, i32 195, i32 196, i32 197, i32 198, i32 199, i32 200, i32 201, i32 202, i32 203, i32 204, i32 205, i32 206, i32 207, i32 208, i32 209, i32 210, i32 211, i32 212, i32 213, i32 214, i32 215, i32 216, i32 217, i32 218, i32 219, i32 220, i32 221, i32 222, i32 223, i32 224, i32 225, i32 226, i32 227, i32 228, i32 229, i32 230, i32 231, i32 232, i32 233, i32 234, i32 235, i32 236, i32 237, i32 238, i32 239, i32 240, i32 241, i32 242, i32 243, i32 244, i32 245, i32 246, i32 247, i32 248, i32 249, i32 250, i32 251, i32 252, i32 253, i32 254, i32 255>9844  %6 = shufflevector <128 x i16> %3, <128 x i16> %4, <256 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127, i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143, i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151, i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159, i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191, i32 192, i32 193, i32 194, i32 195, i32 196, i32 197, i32 198, i32 199, i32 200, i32 201, i32 202, i32 203, i32 204, i32 205, i32 206, i32 207, i32 208, i32 209, i32 210, i32 211, i32 212, i32 213, i32 214, i32 215, i32 216, i32 217, i32 218, i32 219, i32 220, i32 221, i32 222, i32 223, i32 224, i32 225, i32 226, i32 227, i32 228, i32 229, i32 230, i32 231, i32 232, i32 233, i32 234, i32 235, i32 236, i32 237, i32 238, i32 239, i32 240, i32 241, i32 242, i32 243, i32 244, i32 245, i32 246, i32 247, i32 248, i32 249, i32 250, i32 251, i32 252, i32 253, i32 254, i32 255>9845  %7 = shufflevector <256 x i16> %5, <256 x i16> %6, <512 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127, i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143, i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151, i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159, i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191, i32 192, i32 193, i32 194, i32 195, i32 196, i32 197, i32 198, i32 199, i32 200, i32 201, i32 202, i32 203, i32 204, i32 205, i32 206, i32 207, i32 208, i32 209, i32 210, i32 211, i32 212, i32 213, i32 214, i32 215, i32 216, i32 217, i32 218, i32 219, i32 220, i32 221, i32 222, i32 223, i32 224, i32 225, i32 226, i32 227, i32 228, i32 229, i32 230, i32 231, i32 232, i32 233, i32 234, i32 235, i32 236, i32 237, i32 238, i32 239, i32 240, i32 241, i32 242, i32 243, i32 244, i32 245, i32 246, i32 247, i32 248, i32 249, i32 250, i32 251, i32 252, i32 253, i32 254, i32 255, i32 256, i32 257, i32 258, i32 259, i32 260, i32 261, i32 262, i32 263, i32 264, i32 265, i32 266, i32 267, i32 268, i32 269, i32 270, i32 271, i32 272, i32 273, i32 274, i32 275, i32 276, i32 277, i32 278, i32 279, i32 280, i32 281, i32 282, i32 283, i32 284, i32 285, i32 286, i32 287, i32 288, i32 289, i32 290, i32 291, i32 292, i32 293, i32 294, i32 295, i32 296, i32 297, i32 298, i32 299, i32 300, i32 301, i32 302, i32 303, i32 304, i32 305, i32 306, i32 307, i32 308, i32 309, i32 310, i32 311, i32 312, i32 313, i32 314, i32 315, i32 316, i32 317, i32 318, i32 319, i32 320, i32 321, i32 322, i32 323, i32 324, i32 325, i32 326, i32 327, i32 328, i32 329, i32 330, i32 331, i32 332, i32 333, i32 334, i32 335, i32 336, i32 337, i32 338, i32 339, i32 340, i32 341, i32 342, i32 343, i32 344, i32 345, i32 346, i32 347, i32 348, i32 349, i32 350, i32 351, i32 352, i32 353, i32 354, i32 355, i32 356, i32 357, i32 358, i32 359, i32 360, i32 361, i32 362, i32 363, i32 364, i32 365, i32 366, i32 367, i32 368, i32 369, i32 370, i32 371, i32 372, i32 373, i32 374, i32 375, i32 376, i32 377, i32 378, i32 379, i32 380, i32 381, i32 382, i32 383, i32 384, i32 385, i32 386, i32 387, i32 388, i32 389, i32 390, i32 391, i32 392, i32 393, i32 394, i32 395, i32 396, i32 397, i32 398, i32 399, i32 400, i32 401, i32 402, i32 403, i32 404, i32 405, i32 406, i32 407, i32 408, i32 409, i32 410, i32 411, i32 412, i32 413, i32 414, i32 415, i32 416, i32 417, i32 418, i32 419, i32 420, i32 421, i32 422, i32 423, i32 424, i32 425, i32 426, i32 427, i32 428, i32 429, i32 430, i32 431, i32 432, i32 433, i32 434, i32 435, i32 436, i32 437, i32 438, i32 439, i32 440, i32 441, i32 442, i32 443, i32 444, i32 445, i32 446, i32 447, i32 448, i32 449, i32 450, i32 451, i32 452, i32 453, i32 454, i32 455, i32 456, i32 457, i32 458, i32 459, i32 460, i32 461, i32 462, i32 463, i32 464, i32 465, i32 466, i32 467, i32 468, i32 469, i32 470, i32 471, i32 472, i32 473, i32 474, i32 475, i32 476, i32 477, i32 478, i32 479, i32 480, i32 481, i32 482, i32 483, i32 484, i32 485, i32 486, i32 487, i32 488, i32 489, i32 490, i32 491, i32 492, i32 493, i32 494, i32 495, i32 496, i32 497, i32 498, i32 499, i32 500, i32 501, i32 502, i32 503, i32 504, i32 505, i32 506, i32 507, i32 508, i32 509, i32 510, i32 511>9846  %interleaved.vec = shufflevector <512 x i16> %7, <512 x i16> poison, <512 x i32> <i32 0, i32 64, i32 128, i32 192, i32 256, i32 320, i32 384, i32 448, i32 1, i32 65, i32 129, i32 193, i32 257, i32 321, i32 385, i32 449, i32 2, i32 66, i32 130, i32 194, i32 258, i32 322, i32 386, i32 450, i32 3, i32 67, i32 131, i32 195, i32 259, i32 323, i32 387, i32 451, i32 4, i32 68, i32 132, i32 196, i32 260, i32 324, i32 388, i32 452, i32 5, i32 69, i32 133, i32 197, i32 261, i32 325, i32 389, i32 453, i32 6, i32 70, i32 134, i32 198, i32 262, i32 326, i32 390, i32 454, i32 7, i32 71, i32 135, i32 199, i32 263, i32 327, i32 391, i32 455, i32 8, i32 72, i32 136, i32 200, i32 264, i32 328, i32 392, i32 456, i32 9, i32 73, i32 137, i32 201, i32 265, i32 329, i32 393, i32 457, i32 10, i32 74, i32 138, i32 202, i32 266, i32 330, i32 394, i32 458, i32 11, i32 75, i32 139, i32 203, i32 267, i32 331, i32 395, i32 459, i32 12, i32 76, i32 140, i32 204, i32 268, i32 332, i32 396, i32 460, i32 13, i32 77, i32 141, i32 205, i32 269, i32 333, i32 397, i32 461, i32 14, i32 78, i32 142, i32 206, i32 270, i32 334, i32 398, i32 462, i32 15, i32 79, i32 143, i32 207, i32 271, i32 335, i32 399, i32 463, i32 16, i32 80, i32 144, i32 208, i32 272, i32 336, i32 400, i32 464, i32 17, i32 81, i32 145, i32 209, i32 273, i32 337, i32 401, i32 465, i32 18, i32 82, i32 146, i32 210, i32 274, i32 338, i32 402, i32 466, i32 19, i32 83, i32 147, i32 211, i32 275, i32 339, i32 403, i32 467, i32 20, i32 84, i32 148, i32 212, i32 276, i32 340, i32 404, i32 468, i32 21, i32 85, i32 149, i32 213, i32 277, i32 341, i32 405, i32 469, i32 22, i32 86, i32 150, i32 214, i32 278, i32 342, i32 406, i32 470, i32 23, i32 87, i32 151, i32 215, i32 279, i32 343, i32 407, i32 471, i32 24, i32 88, i32 152, i32 216, i32 280, i32 344, i32 408, i32 472, i32 25, i32 89, i32 153, i32 217, i32 281, i32 345, i32 409, i32 473, i32 26, i32 90, i32 154, i32 218, i32 282, i32 346, i32 410, i32 474, i32 27, i32 91, i32 155, i32 219, i32 283, i32 347, i32 411, i32 475, i32 28, i32 92, i32 156, i32 220, i32 284, i32 348, i32 412, i32 476, i32 29, i32 93, i32 157, i32 221, i32 285, i32 349, i32 413, i32 477, i32 30, i32 94, i32 158, i32 222, i32 286, i32 350, i32 414, i32 478, i32 31, i32 95, i32 159, i32 223, i32 287, i32 351, i32 415, i32 479, i32 32, i32 96, i32 160, i32 224, i32 288, i32 352, i32 416, i32 480, i32 33, i32 97, i32 161, i32 225, i32 289, i32 353, i32 417, i32 481, i32 34, i32 98, i32 162, i32 226, i32 290, i32 354, i32 418, i32 482, i32 35, i32 99, i32 163, i32 227, i32 291, i32 355, i32 419, i32 483, i32 36, i32 100, i32 164, i32 228, i32 292, i32 356, i32 420, i32 484, i32 37, i32 101, i32 165, i32 229, i32 293, i32 357, i32 421, i32 485, i32 38, i32 102, i32 166, i32 230, i32 294, i32 358, i32 422, i32 486, i32 39, i32 103, i32 167, i32 231, i32 295, i32 359, i32 423, i32 487, i32 40, i32 104, i32 168, i32 232, i32 296, i32 360, i32 424, i32 488, i32 41, i32 105, i32 169, i32 233, i32 297, i32 361, i32 425, i32 489, i32 42, i32 106, i32 170, i32 234, i32 298, i32 362, i32 426, i32 490, i32 43, i32 107, i32 171, i32 235, i32 299, i32 363, i32 427, i32 491, i32 44, i32 108, i32 172, i32 236, i32 300, i32 364, i32 428, i32 492, i32 45, i32 109, i32 173, i32 237, i32 301, i32 365, i32 429, i32 493, i32 46, i32 110, i32 174, i32 238, i32 302, i32 366, i32 430, i32 494, i32 47, i32 111, i32 175, i32 239, i32 303, i32 367, i32 431, i32 495, i32 48, i32 112, i32 176, i32 240, i32 304, i32 368, i32 432, i32 496, i32 49, i32 113, i32 177, i32 241, i32 305, i32 369, i32 433, i32 497, i32 50, i32 114, i32 178, i32 242, i32 306, i32 370, i32 434, i32 498, i32 51, i32 115, i32 179, i32 243, i32 307, i32 371, i32 435, i32 499, i32 52, i32 116, i32 180, i32 244, i32 308, i32 372, i32 436, i32 500, i32 53, i32 117, i32 181, i32 245, i32 309, i32 373, i32 437, i32 501, i32 54, i32 118, i32 182, i32 246, i32 310, i32 374, i32 438, i32 502, i32 55, i32 119, i32 183, i32 247, i32 311, i32 375, i32 439, i32 503, i32 56, i32 120, i32 184, i32 248, i32 312, i32 376, i32 440, i32 504, i32 57, i32 121, i32 185, i32 249, i32 313, i32 377, i32 441, i32 505, i32 58, i32 122, i32 186, i32 250, i32 314, i32 378, i32 442, i32 506, i32 59, i32 123, i32 187, i32 251, i32 315, i32 379, i32 443, i32 507, i32 60, i32 124, i32 188, i32 252, i32 316, i32 380, i32 444, i32 508, i32 61, i32 125, i32 189, i32 253, i32 317, i32 381, i32 445, i32 509, i32 62, i32 126, i32 190, i32 254, i32 318, i32 382, i32 446, i32 510, i32 63, i32 127, i32 191, i32 255, i32 319, i32 383, i32 447, i32 511>9847  store <512 x i16> %interleaved.vec, ptr %out.vec, align 649848  ret void9849}9850